From 85c554eab560a1e958935dbccdb534a8079f6722 Mon Sep 17 00:00:00 2001 From: Rucha Mahabal Date: Mon, 18 Jul 2022 21:08:15 +0530 Subject: [PATCH 1/6] perf: Replace `db.get_all` with `db.exists` in `is_holiday` - widely used function call, fetching all rows for holidays --- erpnext/hr/doctype/employee/employee.json | 2 +- erpnext/hr/doctype/holiday_list/holiday_list.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/erpnext/hr/doctype/employee/employee.json b/erpnext/hr/doctype/employee/employee.json index d592a9c79e2..ed45e5288c6 100644 --- a/erpnext/hr/doctype/employee/employee.json +++ b/erpnext/hr/doctype/employee/employee.json @@ -813,7 +813,7 @@ "idx": 24, "image_field": "image", "links": [], - "modified": "2021-06-17 11:31:37.730760", + "modified": "2022-07-18 20:03:43.188705", "modified_by": "Administrator", "module": "HR", "name": "Employee", diff --git a/erpnext/hr/doctype/holiday_list/holiday_list.py b/erpnext/hr/doctype/holiday_list/holiday_list.py index ea32ba744d8..de8d578d71c 100644 --- a/erpnext/hr/doctype/holiday_list/holiday_list.py +++ b/erpnext/hr/doctype/holiday_list/holiday_list.py @@ -115,6 +115,6 @@ def is_holiday(holiday_list, date=None): if date is None: date = today() if holiday_list: - return bool(frappe.get_all("Holiday List", dict(name=holiday_list, holiday_date=date))) + return bool(frappe.db.exists("Holiday List", {"name": holiday_list, "holiday_date": date})) else: return False From abb7ac5a0be7a8e48e63186dc8fb02d5656bc1aa Mon Sep 17 00:00:00 2001 From: Rucha Mahabal Date: Mon, 18 Jul 2022 21:13:02 +0530 Subject: [PATCH 2/6] perf: Replace `get_doc` with `db.get_value` in `get_shift_details` --- .../hr/doctype/shift_assignment/shift_assignment.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/erpnext/hr/doctype/shift_assignment/shift_assignment.py b/erpnext/hr/doctype/shift_assignment/shift_assignment.py index 868be6ef719..918b510df27 100644 --- a/erpnext/hr/doctype/shift_assignment/shift_assignment.py +++ b/erpnext/hr/doctype/shift_assignment/shift_assignment.py @@ -294,7 +294,18 @@ def get_shift_details(shift_type_name, for_date=None): return None if not for_date: for_date = nowdate() - shift_type = frappe.get_doc("Shift Type", shift_type_name) + shift_type = frappe.db.get_value( + "Shift Type", + shift_type_name, + [ + "name", + "start_time", + "end_time", + "begin_check_in_before_shift_start_time", + "allow_check_out_after_shift_end_time", + ], + as_dict=1, + ) start_datetime = datetime.combine(for_date, datetime.min.time()) + shift_type.start_time for_date = ( for_date + timedelta(days=1) if shift_type.start_time > shift_type.end_time else for_date From 711501be5e1f6b2fe39529d8287d912f0ccea9f2 Mon Sep 17 00:00:00 2001 From: Rucha Mahabal Date: Mon, 18 Jul 2022 21:32:04 +0530 Subject: [PATCH 3/6] fix: Remove unnecessary list comprehensions --- erpnext/hr/doctype/shift_type/shift_type.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/erpnext/hr/doctype/shift_type/shift_type.py b/erpnext/hr/doctype/shift_type/shift_type.py index 2000eeb5443..e4a40c0807f 100644 --- a/erpnext/hr/doctype/shift_type/shift_type.py +++ b/erpnext/hr/doctype/shift_type/shift_type.py @@ -156,21 +156,19 @@ class ShiftType(Document): if not from_date: del filters["start_date"] - assigned_employees = frappe.get_all("Shift Assignment", "employee", filters, as_list=True) - assigned_employees = [x[0] for x in assigned_employees] + assigned_employees = frappe.get_all("Shift Assignment", filters, pluck="employee") if consider_default_shift: filters = {"default_shift": self.name, "status": ["!=", "Inactive"]} - default_shift_employees = frappe.get_all("Employee", "name", filters, as_list=True) - default_shift_employees = [x[0] for x in default_shift_employees] + default_shift_employees = frappe.get_all("Employee", filters, pluck="name") return list(set(assigned_employees + default_shift_employees)) return assigned_employees def process_auto_attendance_for_all_shifts(): - shift_list = frappe.get_all("Shift Type", "name", {"enable_auto_attendance": "1"}, as_list=True) + shift_list = frappe.get_all("Shift Type", filters={"enable_auto_attendance": "1"}, pluck="name") for shift in shift_list: - doc = frappe.get_doc("Shift Type", shift[0]) + doc = frappe.get_doc("Shift Type", shift) doc.process_auto_attendance() From 8c9035d914406e3134e9516b7194d4c50662f37d Mon Sep 17 00:00:00 2001 From: Rucha Mahabal Date: Tue, 19 Jul 2022 14:47:14 +0530 Subject: [PATCH 4/6] perf: Use `get_cached_value` and `get_cached_doc` --- erpnext/hr/doctype/employee/employee.py | 4 +++- erpnext/hr/doctype/employee_checkin/employee_checkin.py | 4 ++-- erpnext/hr/doctype/holiday_list/holiday_list.py | 4 +++- erpnext/hr/doctype/shift_assignment/shift_assignment.py | 6 +++--- erpnext/hr/doctype/shift_type/shift_type.py | 4 ++-- 5 files changed, 13 insertions(+), 9 deletions(-) diff --git a/erpnext/hr/doctype/employee/employee.py b/erpnext/hr/doctype/employee/employee.py index d24e7038422..8aaae5956b0 100755 --- a/erpnext/hr/doctype/employee/employee.py +++ b/erpnext/hr/doctype/employee/employee.py @@ -349,7 +349,9 @@ def get_employee_email(employee_doc): def get_holiday_list_for_employee(employee, raise_exception=True): if employee: - holiday_list, company = frappe.db.get_value("Employee", employee, ["holiday_list", "company"]) + holiday_list, company = frappe.get_cached_value( + "Employee", employee, ["holiday_list", "company"] + ) else: holiday_list = "" company = frappe.db.get_value("Global Defaults", None, "default_company") diff --git a/erpnext/hr/doctype/employee_checkin/employee_checkin.py b/erpnext/hr/doctype/employee_checkin/employee_checkin.py index 58618b6358c..8b0a60e20e9 100644 --- a/erpnext/hr/doctype/employee_checkin/employee_checkin.py +++ b/erpnext/hr/doctype/employee_checkin/employee_checkin.py @@ -131,7 +131,7 @@ def mark_attendance_and_link_log( return None elif attendance_status in ("Present", "Absent", "Half Day"): - employee_doc = frappe.get_doc("Employee", employee) + company = frappe.get_cached_value("Employee", employee, "company") duplicate = frappe.db.exists( "Attendance", {"employee": employee, "attendance_date": attendance_date, "docstatus": ("!=", "2")}, @@ -144,7 +144,7 @@ def mark_attendance_and_link_log( "attendance_date": attendance_date, "status": attendance_status, "working_hours": working_hours, - "company": employee_doc.company, + "company": company, "shift": shift, "late_entry": late_entry, "early_exit": early_exit, diff --git a/erpnext/hr/doctype/holiday_list/holiday_list.py b/erpnext/hr/doctype/holiday_list/holiday_list.py index de8d578d71c..1cf44536b59 100644 --- a/erpnext/hr/doctype/holiday_list/holiday_list.py +++ b/erpnext/hr/doctype/holiday_list/holiday_list.py @@ -115,6 +115,8 @@ def is_holiday(holiday_list, date=None): if date is None: date = today() if holiday_list: - return bool(frappe.db.exists("Holiday List", {"name": holiday_list, "holiday_date": date})) + return bool( + frappe.db.exists("Holiday", {"parent": holiday_list, "holiday_date": date}, cache=True) + ) else: return False diff --git a/erpnext/hr/doctype/shift_assignment/shift_assignment.py b/erpnext/hr/doctype/shift_assignment/shift_assignment.py index 918b510df27..483d0fb1c41 100644 --- a/erpnext/hr/doctype/shift_assignment/shift_assignment.py +++ b/erpnext/hr/doctype/shift_assignment/shift_assignment.py @@ -169,7 +169,7 @@ def get_employee_shift( """ if for_date is None: for_date = nowdate() - default_shift = frappe.db.get_value("Employee", employee, "default_shift") + default_shift = frappe.get_cached_value("Employee", employee, "default_shift") shift_type_name = None shift_assignment_details = frappe.db.get_value( "Shift Assignment", @@ -187,7 +187,7 @@ def get_employee_shift( if not shift_type_name and consider_default_shift: shift_type_name = default_shift if shift_type_name: - holiday_list_name = frappe.db.get_value("Shift Type", shift_type_name, "holiday_list") + holiday_list_name = frappe.get_cached_value("Shift Type", shift_type_name, "holiday_list") if not holiday_list_name: holiday_list_name = get_holiday_list_for_employee(employee, False) if holiday_list_name and is_holiday(holiday_list_name, for_date): @@ -294,7 +294,7 @@ def get_shift_details(shift_type_name, for_date=None): return None if not for_date: for_date = nowdate() - shift_type = frappe.db.get_value( + shift_type = frappe.get_cached_value( "Shift Type", shift_type_name, [ diff --git a/erpnext/hr/doctype/shift_type/shift_type.py b/erpnext/hr/doctype/shift_type/shift_type.py index e4a40c0807f..791c791ad07 100644 --- a/erpnext/hr/doctype/shift_type/shift_type.py +++ b/erpnext/hr/doctype/shift_type/shift_type.py @@ -107,7 +107,7 @@ class ShiftType(Document): """Marks Absents for the given employee on working days in this shift which have no attendance marked. The Absent is marked starting from 'process_attendance_after' or employee creation date. """ - date_of_joining, relieving_date, employee_creation = frappe.db.get_value( + date_of_joining, relieving_date, employee_creation = frappe.get_cached_value( "Employee", employee, ["date_of_joining", "relieving_date", "creation"] ) if not date_of_joining: @@ -168,7 +168,7 @@ class ShiftType(Document): def process_auto_attendance_for_all_shifts(): shift_list = frappe.get_all("Shift Type", filters={"enable_auto_attendance": "1"}, pluck="name") for shift in shift_list: - doc = frappe.get_doc("Shift Type", shift) + doc = frappe.get_cached_doc("Shift Type", shift) doc.process_auto_attendance() From 6938025952d5d702c2e28b5086f18c7c9cb3b41a Mon Sep 17 00:00:00 2001 From: Rucha Mahabal Date: Tue, 19 Jul 2022 16:02:31 +0530 Subject: [PATCH 5/6] perf: index shift type and employee in checkins and assignment to avoid full table scans --- erpnext/hr/doctype/employee_checkin/employee_checkin.json | 8 +++++--- erpnext/hr/doctype/shift_assignment/shift_assignment.json | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/erpnext/hr/doctype/employee_checkin/employee_checkin.json b/erpnext/hr/doctype/employee_checkin/employee_checkin.json index d34316dc0f3..19ef1b3c0ea 100644 --- a/erpnext/hr/doctype/employee_checkin/employee_checkin.json +++ b/erpnext/hr/doctype/employee_checkin/employee_checkin.json @@ -26,7 +26,8 @@ "fieldtype": "Link", "label": "Employee", "options": "Employee", - "reqd": 1 + "reqd": 1, + "search_index": 1 }, { "fetch_from": "employee.employee_name", @@ -48,7 +49,8 @@ "fieldtype": "Link", "label": "Shift", "options": "Shift Type", - "read_only": 1 + "read_only": 1, + "search_index": 1 }, { "fieldname": "column_break_4", @@ -107,7 +109,7 @@ } ], "links": [], - "modified": "2020-07-08 11:02:32.660986", + "modified": "2022-07-19 15:38:41.767539", "modified_by": "Administrator", "module": "HR", "name": "Employee Checkin", diff --git a/erpnext/hr/doctype/shift_assignment/shift_assignment.json b/erpnext/hr/doctype/shift_assignment/shift_assignment.json index ce2a10f229f..c2df00be628 100644 --- a/erpnext/hr/doctype/shift_assignment/shift_assignment.json +++ b/erpnext/hr/doctype/shift_assignment/shift_assignment.json @@ -25,7 +25,8 @@ "fieldtype": "Link", "label": "Employee", "options": "Employee", - "reqd": 1 + "reqd": 1, + "search_index": 1 }, { "fetch_from": "employee.employee_name", @@ -48,7 +49,8 @@ "in_list_view": 1, "label": "Shift Type", "options": "Shift Type", - "reqd": 1 + "reqd": 1, + "search_index": 1 }, { "fieldname": "column_break_3", @@ -105,7 +107,7 @@ ], "is_submittable": 1, "links": [], - "modified": "2020-06-15 14:27:54.310773", + "modified": "2022-07-19 15:27:54.310773", "modified_by": "Administrator", "module": "HR", "name": "Shift Assignment", From 8eb9aaafe9850913b05e3983e9ae5361f049f00c Mon Sep 17 00:00:00 2001 From: Rucha Mahabal Date: Tue, 19 Jul 2022 16:07:18 +0530 Subject: [PATCH 6/6] fix: move auto attendance job to long queue --- erpnext/hooks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/erpnext/hooks.py b/erpnext/hooks.py index 386ac0a2888..cf1714a25e1 100644 --- a/erpnext/hooks.py +++ b/erpnext/hooks.py @@ -483,12 +483,12 @@ scheduler_events = { "erpnext.erpnext_integrations.doctype.plaid_settings.plaid_settings.automatic_synchronization", "erpnext.projects.doctype.project.project.hourly_reminder", "erpnext.projects.doctype.project.project.collect_project_status", - "erpnext.hr.doctype.shift_type.shift_type.process_auto_attendance_for_all_shifts", "erpnext.support.doctype.issue.issue.set_service_level_agreement_variance", "erpnext.erpnext_integrations.connectors.shopify_connection.sync_old_orders", ], "hourly_long": [ - "erpnext.stock.doctype.repost_item_valuation.repost_item_valuation.repost_entries" + "erpnext.stock.doctype.repost_item_valuation.repost_item_valuation.repost_entries", + "erpnext.hr.doctype.shift_type.shift_type.process_auto_attendance_for_all_shifts", ], "daily": [ "erpnext.support.doctype.issue.issue.auto_close_tickets",