From 0d65d878deb717b80a22cae4d152d07289acae58 Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Sat, 3 Feb 2024 11:58:11 +0530 Subject: [PATCH 01/28] refactor: more options for 'status' and move it to top --- .../transaction_deletion_record.json | 22 ++++++++++++++----- .../transaction_deletion_record.py | 2 +- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json index 23e59472a6d..8f3a5d05666 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json @@ -7,10 +7,12 @@ "engine": "InnoDB", "field_order": [ "company", + "column_break_txbg", + "status", + "section_break_tbej", "doctypes", "doctypes_to_be_ignored", - "amended_from", - "status" + "amended_from" ], "fields": [ { @@ -46,18 +48,27 @@ { "fieldname": "status", "fieldtype": "Select", - "hidden": 1, "label": "Status", - "options": "Draft\nCompleted" + "options": "Queued\nRunning\nFailed\nCompleted\nCancelled", + "read_only": 1 + }, + { + "fieldname": "column_break_txbg", + "fieldtype": "Column Break" + }, + { + "fieldname": "section_break_tbej", + "fieldtype": "Section Break" } ], "index_web_pages_for_search": 1, "is_submittable": 1, "links": [], - "modified": "2021-08-04 20:15:59.071493", + "modified": "2024-02-03 12:42:21.628177", "modified_by": "Administrator", "module": "Setup", "name": "Transaction Deletion Record", + "naming_rule": "Expression (old style)", "owner": "Administrator", "permissions": [ { @@ -76,5 +87,6 @@ ], "sort_field": "modified", "sort_order": "DESC", + "states": [], "track_changes": 1 } \ No newline at end of file diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py index 88c4b078977..cbe7e05adb4 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py @@ -26,7 +26,7 @@ class TransactionDeletionRecord(Document): company: DF.Link doctypes: DF.Table[TransactionDeletionRecordItem] doctypes_to_be_ignored: DF.Table[TransactionDeletionRecordItem] - status: DF.Literal["Draft", "Completed"] + status: DF.Literal["Queued", "Running", "Completed"] # end: auto-generated types def __init__(self, *args, **kwargs): From 6fbb67b1d2da1fe9aa1879ce354f5702150584c8 Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Sat, 3 Feb 2024 12:46:12 +0530 Subject: [PATCH 02/28] refactor: set status and trigger job on submit --- .../transaction_deletion_record.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py index cbe7e05adb4..073eec29a36 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py @@ -26,7 +26,7 @@ class TransactionDeletionRecord(Document): company: DF.Link doctypes: DF.Table[TransactionDeletionRecordItem] doctypes_to_be_ignored: DF.Table[TransactionDeletionRecordItem] - status: DF.Literal["Queued", "Running", "Completed"] + status: DF.Literal["Queued", "Running", "Failed", "Completed", "Cancelled"] # end: auto-generated types def __init__(self, *args, **kwargs): @@ -52,6 +52,16 @@ class TransactionDeletionRecord(Document): if not self.doctypes_to_be_ignored: self.populate_doctypes_to_be_ignored_table() + def before_save(self): + self.status = "" + + def on_submit(self): + self.db_set("status", "Queued") + + def on_cancel(self): + self.db_set("status", "Cancelled") + + def start_deletion_process(self): self.delete_bins() self.delete_lead_addresses() self.reset_company_values() From d0dc2c6e77c4e04e8d74a36a632ad6f2189dfedc Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Sat, 3 Feb 2024 14:29:04 +0530 Subject: [PATCH 03/28] refactor: tasks section and UI niceties --- .../transaction_deletion_record.json | 53 ++++++++++++++++++- .../transaction_deletion_record.py | 19 +++++++ 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json index 8f3a5d05666..a9e04d3892e 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json @@ -9,6 +9,12 @@ "company", "column_break_txbg", "status", + "tasks_section", + "delete_bin_data", + "delete_leads_and_addresses", + "reset_company_default_values", + "clear_notifications", + "delete_transactions", "section_break_tbej", "doctypes", "doctypes_to_be_ignored", @@ -59,12 +65,57 @@ { "fieldname": "section_break_tbej", "fieldtype": "Section Break" + }, + { + "fieldname": "tasks_section", + "fieldtype": "Section Break", + "label": "Tasks" + }, + { + "default": "0", + "fieldname": "delete_bin_data", + "fieldtype": "Check", + "label": "Delete Bins", + "no_copy": 1, + "read_only": 1 + }, + { + "default": "0", + "fieldname": "delete_leads_and_addresses", + "fieldtype": "Check", + "label": "Delete Leads and Addresses", + "no_copy": 1, + "read_only": 1 + }, + { + "default": "0", + "fieldname": "clear_notifications", + "fieldtype": "Check", + "label": "Clear Notifications", + "no_copy": 1, + "read_only": 1 + }, + { + "default": "0", + "fieldname": "reset_company_default_values", + "fieldtype": "Check", + "label": "Reset Company Default Values", + "no_copy": 1, + "read_only": 1 + }, + { + "default": "0", + "fieldname": "delete_transactions", + "fieldtype": "Check", + "label": "Delete Transactions", + "no_copy": 1, + "read_only": 1 } ], "index_web_pages_for_search": 1, "is_submittable": 1, "links": [], - "modified": "2024-02-03 12:42:21.628177", + "modified": "2024-02-03 14:40:40.207482", "modified_by": "Administrator", "module": "Setup", "name": "Transaction Deletion Record", diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py index 073eec29a36..f1f6a5b90d3 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py @@ -23,9 +23,14 @@ class TransactionDeletionRecord(Document): ) amended_from: DF.Link | None + clear_notifications: DF.Check company: DF.Link + delete_bin_data: DF.Check + delete_leads_and_addresses: DF.Check + delete_transactions: DF.Check doctypes: DF.Table[TransactionDeletionRecordItem] doctypes_to_be_ignored: DF.Table[TransactionDeletionRecordItem] + reset_company_default_values: DF.Check status: DF.Literal["Queued", "Running", "Failed", "Completed", "Cancelled"] # end: auto-generated types @@ -52,8 +57,16 @@ class TransactionDeletionRecord(Document): if not self.doctypes_to_be_ignored: self.populate_doctypes_to_be_ignored_table() + def reset_task_flags(self): + self.clear_notifications = 0 + self.delete_bin_data = 0 + self.delete_leads_and_addresses = 0 + self.delete_transactions = 0 + self.reset_company_default_values = 0 + def before_save(self): self.status = "" + self.reset_task_flags() def on_submit(self): self.db_set("status", "Queued") @@ -61,11 +74,13 @@ class TransactionDeletionRecord(Document): def on_cancel(self): self.db_set("status", "Cancelled") + @frappe.whitelist() def start_deletion_process(self): self.delete_bins() self.delete_lead_addresses() self.reset_company_values() clear_notifications() + self.db_set("clear_notifications", 1) self.delete_company_transactions() def populate_doctypes_to_be_ignored_table(self): @@ -79,6 +94,7 @@ class TransactionDeletionRecord(Document): (select name from tabWarehouse where company=%s)""", self.company, ) + self.db_set("delete_bin_data", 1) def delete_lead_addresses(self): """Delete addresses to which leads are linked""" @@ -117,12 +133,14 @@ class TransactionDeletionRecord(Document): leads=",".join(leads) ) ) + self.db_set("delete_leads_and_addresses", 1) def reset_company_values(self): company_obj = frappe.get_doc("Company", self.company) company_obj.total_monthly_sales = 0 company_obj.sales_monthly_history = None company_obj.save() + self.db_set("reset_company_default_values", 1) def delete_company_transactions(self): doctypes_to_be_ignored_list = self.get_doctypes_to_be_ignored_list() @@ -156,6 +174,7 @@ class TransactionDeletionRecord(Document): if naming_series: if "#" in naming_series: self.update_naming_series(naming_series, docfield["parent"]) + self.db_set("delete_transactions", 1) def get_doctypes_to_be_ignored_list(self): singles = frappe.get_all("DocType", filters={"issingle": 1}, pluck="name") From 8944ab8b6ad1da2acdd5a852a17fc72e11d84695 Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Sat, 3 Feb 2024 14:49:18 +0530 Subject: [PATCH 04/28] refactor: UI trigger --- .../transaction_deletion_record.js | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.js b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.js index 527c753d6a9..671f927106d 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.js +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.js @@ -24,6 +24,17 @@ frappe.ui.form.on("Transaction Deletion Record", { refresh: function (frm) { frm.fields_dict["doctypes_to_be_ignored"].grid.set_column_disp("no_of_docs", false); frm.refresh_field("doctypes_to_be_ignored"); + + if (frm.doc.docstatus==1 && ['Queued', 'Failed'].find(x => x == frm.doc.status)) { + let execute_btn = __("Start / Resume") + + frm.add_custom_button(execute_btn, () => { + frm.call({ + method: 'start_deletion_process', + doc: frm.doc + }); + }); + } }, }); From 6a77d86a53580b670937b92bcbde69ea920dbb9e Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Sat, 3 Feb 2024 20:21:30 +0530 Subject: [PATCH 05/28] refactor: use flags to decide on current stage --- .../transaction_deletion_record.js | 12 +- .../transaction_deletion_record.json | 10 +- .../transaction_deletion_record.py | 196 ++++++++++-------- .../transaction_deletion_record_item.json | 19 +- .../transaction_deletion_record_item.py | 2 + 5 files changed, 141 insertions(+), 98 deletions(-) diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.js b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.js index 671f927106d..1a8b52f46bd 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.js +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.js @@ -16,9 +16,15 @@ frappe.ui.form.on("Transaction Deletion Record", { }); } - frm.get_field("doctypes_to_be_ignored").grid.cannot_add_rows = true; - frm.fields_dict["doctypes_to_be_ignored"].grid.set_column_disp("no_of_docs", false); - frm.refresh_field("doctypes_to_be_ignored"); + + frm.get_field('doctypes_to_be_ignored').grid.cannot_add_rows = true; + frm.fields_dict['doctypes_to_be_ignored'].grid.set_column_disp('no_of_docs', false); + frm.fields_dict['doctypes_to_be_ignored'].grid.set_column_disp('done', false); + frm.refresh_field('doctypes_to_be_ignored'); + + frm.get_field('doctypes').grid.cannot_add_rows = true; + frm.fields_dict['doctypes'].grid.set_column_disp('no_of_docs', true); + frm.refresh_field('doctypes'); }, refresh: function (frm) { diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json index a9e04d3892e..6a848413ffc 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json @@ -15,6 +15,7 @@ "reset_company_default_values", "clear_notifications", "delete_transactions", + "initialize_doctypes_table", "section_break_tbej", "doctypes", "doctypes_to_be_ignored", @@ -110,12 +111,19 @@ "label": "Delete Transactions", "no_copy": 1, "read_only": 1 + }, + { + "default": "0", + "fieldname": "initialize_doctypes_table", + "fieldtype": "Check", + "label": "Initialize Summary Table", + "read_only": 1 } ], "index_web_pages_for_search": 1, "is_submittable": 1, "links": [], - "modified": "2024-02-03 14:40:40.207482", + "modified": "2024-02-03 20:48:34.107577", "modified_by": "Administrator", "module": "Setup", "name": "Transaction Deletion Record", diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py index f1f6a5b90d3..5a5a827d304 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py @@ -30,6 +30,7 @@ class TransactionDeletionRecord(Document): delete_transactions: DF.Check doctypes: DF.Table[TransactionDeletionRecordItem] doctypes_to_be_ignored: DF.Table[TransactionDeletionRecordItem] + initialize_doctypes_table: DF.Check reset_company_default_values: DF.Check status: DF.Literal["Queued", "Running", "Failed", "Completed", "Cancelled"] # end: auto-generated types @@ -79,8 +80,10 @@ class TransactionDeletionRecord(Document): self.delete_bins() self.delete_lead_addresses() self.reset_company_values() - clear_notifications() - self.db_set("clear_notifications", 1) + if not self.clear_notifications: + clear_notifications() + self.db_set("clear_notifications", 1) + self.initialize_doctypes_to_be_deleted_table() self.delete_company_transactions() def populate_doctypes_to_be_ignored_table(self): @@ -89,92 +92,108 @@ class TransactionDeletionRecord(Document): self.append("doctypes_to_be_ignored", {"doctype_name": doctype}) def delete_bins(self): - frappe.db.sql( - """delete from `tabBin` where warehouse in - (select name from tabWarehouse where company=%s)""", - self.company, - ) - self.db_set("delete_bin_data", 1) + if not self.delete_bin_data: + frappe.db.sql( + """delete from `tabBin` where warehouse in + (select name from tabWarehouse where company=%s)""", + self.company, + ) + self.db_set("delete_bin_data", 1) def delete_lead_addresses(self): """Delete addresses to which leads are linked""" - leads = frappe.get_all("Lead", filters={"company": self.company}) - leads = ["'%s'" % row.get("name") for row in leads] - addresses = [] - if leads: - addresses = frappe.db.sql_list( - """select parent from `tabDynamic Link` where link_name - in ({leads})""".format( - leads=",".join(leads) - ) - ) - - if addresses: - addresses = ["%s" % frappe.db.escape(addr) for addr in addresses] - - frappe.db.sql( - """delete from `tabAddress` where name in ({addresses}) and - name not in (select distinct dl1.parent from `tabDynamic Link` dl1 - inner join `tabDynamic Link` dl2 on dl1.parent=dl2.parent - and dl1.link_doctype<>dl2.link_doctype)""".format( - addresses=",".join(addresses) - ) - ) - - frappe.db.sql( - """delete from `tabDynamic Link` where link_doctype='Lead' - and parenttype='Address' and link_name in ({leads})""".format( + if not self.delete_leads_and_addresses: + leads = frappe.get_all("Lead", filters={"company": self.company}) + leads = ["'%s'" % row.get("name") for row in leads] + addresses = [] + if leads: + addresses = frappe.db.sql_list( + """select parent from `tabDynamic Link` where link_name + in ({leads})""".format( leads=",".join(leads) ) ) - frappe.db.sql( - """update `tabCustomer` set lead_name=NULL where lead_name in ({leads})""".format( - leads=",".join(leads) + if addresses: + addresses = ["%s" % frappe.db.escape(addr) for addr in addresses] + + frappe.db.sql( + """delete from `tabAddress` where name in ({addresses}) and + name not in (select distinct dl1.parent from `tabDynamic Link` dl1 + inner join `tabDynamic Link` dl2 on dl1.parent=dl2.parent + and dl1.link_doctype<>dl2.link_doctype)""".format( + addresses=",".join(addresses) + ) + ) + + frappe.db.sql( + """delete from `tabDynamic Link` where link_doctype='Lead' + and parenttype='Address' and link_name in ({leads})""".format( + leads=",".join(leads) + ) + ) + + frappe.db.sql( + """update `tabCustomer` set lead_name=NULL where lead_name in ({leads})""".format( + leads=",".join(leads) + ) ) - ) - self.db_set("delete_leads_and_addresses", 1) + self.db_set("delete_leads_and_addresses", 1) def reset_company_values(self): - company_obj = frappe.get_doc("Company", self.company) - company_obj.total_monthly_sales = 0 - company_obj.sales_monthly_history = None - company_obj.save() - self.db_set("reset_company_default_values", 1) + if not self.reset_company_default_values: + company_obj = frappe.get_doc("Company", self.company) + company_obj.total_monthly_sales = 0 + company_obj.sales_monthly_history = None + company_obj.save() + self.db_set("reset_company_default_values", 1) + + def initialize_doctypes_to_be_deleted_table(self): + if not self.initialize_doctypes_table: + doctypes_to_be_ignored_list = self.get_doctypes_to_be_ignored_list() + docfields = self.get_doctypes_with_company_field(doctypes_to_be_ignored_list) + tables = self.get_all_child_doctypes() + for docfield in docfields: + if docfield["parent"] != self.doctype: + no_of_docs = self.get_number_of_docs_linked_with_specified_company( + docfield["parent"], docfield["fieldname"] + ) + if no_of_docs > 0: + # Initialize + self.populate_doctypes_table(tables, docfield["parent"], docfield["fieldname"], 0) + self.db_set("initialize_doctypes_table", 1) def delete_company_transactions(self): - doctypes_to_be_ignored_list = self.get_doctypes_to_be_ignored_list() - docfields = self.get_doctypes_with_company_field(doctypes_to_be_ignored_list) + if not self.delete_transactions: + doctypes_to_be_ignored_list = self.get_doctypes_to_be_ignored_list() + docfields = self.get_doctypes_with_company_field(doctypes_to_be_ignored_list) - tables = self.get_all_child_doctypes() - for docfield in docfields: - if docfield["parent"] != self.doctype: - no_of_docs = self.get_number_of_docs_linked_with_specified_company( - docfield["parent"], docfield["fieldname"] - ) - - if no_of_docs > 0: - self.delete_version_log(docfield["parent"], docfield["fieldname"]) - - reference_docs = frappe.get_all( - docfield["parent"], filters={docfield["fieldname"]: self.company} + tables = self.get_all_child_doctypes() + for docfield in self.doctypes: + if docfield.doctype_name != self.doctype: + no_of_docs = self.get_number_of_docs_linked_with_specified_company( + docfield.doctype_name, docfield.docfield_name ) - reference_doc_names = [r.name for r in reference_docs] + if no_of_docs > 0: + reference_docs = frappe.get_all( + docfield.doctype_name, filters={docfield.docfield_name: self.company}, limit=self.batch_size + ) + reference_doc_names = [r.name for r in reference_docs] - self.delete_communications(docfield["parent"], reference_doc_names) - self.delete_comments(docfield["parent"], reference_doc_names) - self.unlink_attachments(docfield["parent"], reference_doc_names) + self.delete_version_log(docfield.doctype_name, reference_doc_names) + self.delete_communications(docfield.doctype_name, reference_doc_names) + self.delete_comments(docfield.doctype_name, reference_doc_names) + self.unlink_attachments(docfield.doctype_name, reference_doc_names) - self.populate_doctypes_table(tables, docfield["parent"], no_of_docs) + self.delete_child_tables(docfield.doctype_name, reference_doc_names) + self.delete_docs_linked_with_specified_company(docfield.doctype_name, docfield.docfield_name) - self.delete_child_tables(docfield["parent"], docfield["fieldname"]) - self.delete_docs_linked_with_specified_company(docfield["parent"], docfield["fieldname"]) - - naming_series = frappe.db.get_value("DocType", docfield["parent"], "autoname") - if naming_series: - if "#" in naming_series: - self.update_naming_series(naming_series, docfield["parent"]) - self.db_set("delete_transactions", 1) + naming_series = frappe.db.get_value("DocType", docfield.doctype_name, "autoname") + # TODO: do this at the end of each doctype + if naming_series: + if "#" in naming_series: + self.update_naming_series(naming_series, docfield.doctype_name) + self.db_set("delete_transactions", 1) def get_doctypes_to_be_ignored_list(self): singles = frappe.get_all("DocType", filters={"issingle": 1}, pluck="name") @@ -203,22 +222,21 @@ class TransactionDeletionRecord(Document): def get_number_of_docs_linked_with_specified_company(self, doctype, company_fieldname): return frappe.db.count(doctype, {company_fieldname: self.company}) - def populate_doctypes_table(self, tables, doctype, no_of_docs): + def populate_doctypes_table(self, tables, doctype, fieldname, no_of_docs): + self.flags.ignore_validate_update_after_submit = True if doctype not in tables: - self.append("doctypes", {"doctype_name": doctype, "no_of_docs": no_of_docs}) - - def delete_child_tables(self, doctype, company_fieldname): - parent_docs_to_be_deleted = frappe.get_all( - doctype, {company_fieldname: self.company}, pluck="name" - ) + self.append( + "doctypes", {"doctype_name": doctype, "docfield_name": fieldname, "no_of_docs": no_of_docs} + ) + self.save(ignore_permissions=True) + def delete_child_tables(self, doctype, reference_doc_names): child_tables = frappe.get_all( "DocField", filters={"fieldtype": "Table", "parent": doctype}, pluck="options" ) - for batch in create_batch(parent_docs_to_be_deleted, self.batch_size): - for table in child_tables: - frappe.db.delete(table, {"parent": ["in", batch]}) + for table in child_tables: + frappe.db.delete(table, {"parent": ["in", reference_doc_names]}) def delete_docs_linked_with_specified_company(self, doctype, company_fieldname): frappe.db.delete(doctype, {company_fieldname: self.company}) @@ -242,17 +260,11 @@ class TransactionDeletionRecord(Document): frappe.db.sql("""update `tabSeries` set current = %s where name=%s""", (last, prefix)) - def delete_version_log(self, doctype, company_fieldname): - dt = qb.DocType(doctype) - names = qb.from_(dt).select(dt.name).where(dt[company_fieldname] == self.company).run(as_list=1) - names = [x[0] for x in names] - - if names: - versions = qb.DocType("Version") - for batch in create_batch(names, self.batch_size): - qb.from_(versions).delete().where( - (versions.ref_doctype == doctype) & (versions.docname.isin(batch)) - ).run() + def delete_version_log(self, doctype, docnames): + versions = qb.DocType("Version") + qb.from_(versions).delete().where( + (versions.ref_doctype == doctype) & (versions.docname.isin(docnames)) + ).run() def delete_communications(self, doctype, reference_doc_names): communications = frappe.get_all( diff --git a/erpnext/setup/doctype/transaction_deletion_record_item/transaction_deletion_record_item.json b/erpnext/setup/doctype/transaction_deletion_record_item/transaction_deletion_record_item.json index be0be945c4e..4e5e1846999 100644 --- a/erpnext/setup/doctype/transaction_deletion_record_item/transaction_deletion_record_item.json +++ b/erpnext/setup/doctype/transaction_deletion_record_item/transaction_deletion_record_item.json @@ -6,7 +6,9 @@ "engine": "InnoDB", "field_order": [ "doctype_name", - "no_of_docs" + "docfield_name", + "no_of_docs", + "done" ], "fields": [ { @@ -22,12 +24,24 @@ "fieldtype": "Data", "in_list_view": 1, "label": "Number of Docs" + }, + { + "default": "0", + "fieldname": "done", + "fieldtype": "Check", + "in_list_view": 1, + "label": "Done" + }, + { + "fieldname": "docfield_name", + "fieldtype": "Data", + "label": "DocField Name" } ], "index_web_pages_for_search": 1, "istable": 1, "links": [], - "modified": "2021-05-08 23:10:46.166744", + "modified": "2024-02-03 21:06:32.274445", "modified_by": "Administrator", "module": "Setup", "name": "Transaction Deletion Record Item", @@ -35,5 +49,6 @@ "permissions": [], "sort_field": "modified", "sort_order": "DESC", + "states": [], "track_changes": 1 } \ No newline at end of file diff --git a/erpnext/setup/doctype/transaction_deletion_record_item/transaction_deletion_record_item.py b/erpnext/setup/doctype/transaction_deletion_record_item/transaction_deletion_record_item.py index f154cdb2474..ce716ac477c 100644 --- a/erpnext/setup/doctype/transaction_deletion_record_item/transaction_deletion_record_item.py +++ b/erpnext/setup/doctype/transaction_deletion_record_item/transaction_deletion_record_item.py @@ -15,7 +15,9 @@ class TransactionDeletionRecordItem(Document): if TYPE_CHECKING: from frappe.types import DF + docfield_name: DF.Data | None doctype_name: DF.Link + done: DF.Check no_of_docs: DF.Data | None parent: DF.Data parentfield: DF.Data From cccb2d5141e30234dd9d3f7ff877c9aaaa44879e Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Sun, 4 Feb 2024 08:09:53 +0530 Subject: [PATCH 06/28] refactor: reorder flags in Tasks section --- .../transaction_deletion_record.json | 4 ++-- .../transaction_deletion_record.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json index 6a848413ffc..dc35fe59553 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json @@ -14,8 +14,8 @@ "delete_leads_and_addresses", "reset_company_default_values", "clear_notifications", - "delete_transactions", "initialize_doctypes_table", + "delete_transactions", "section_break_tbej", "doctypes", "doctypes_to_be_ignored", @@ -123,7 +123,7 @@ "index_web_pages_for_search": 1, "is_submittable": 1, "links": [], - "modified": "2024-02-03 20:48:34.107577", + "modified": "2024-02-04 08:09:26.784109", "modified_by": "Administrator", "module": "Setup", "name": "Transaction Deletion Record", diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py index 5a5a827d304..96e5bf97891 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py @@ -170,7 +170,7 @@ class TransactionDeletionRecord(Document): tables = self.get_all_child_doctypes() for docfield in self.doctypes: - if docfield.doctype_name != self.doctype: + if docfield.doctype_name != self.doctype and not docfield.done: no_of_docs = self.get_number_of_docs_linked_with_specified_company( docfield.doctype_name, docfield.docfield_name ) @@ -193,6 +193,9 @@ class TransactionDeletionRecord(Document): if naming_series: if "#" in naming_series: self.update_naming_series(naming_series, docfield.doctype_name) + + else: + frappe.db.set_value(docfield.doctype, docfield.name, "done", 1) self.db_set("delete_transactions", 1) def get_doctypes_to_be_ignored_list(self): From b12ca65fcc76c85c9a26fb9d6d2e252ede4a3e70 Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Sun, 4 Feb 2024 08:25:18 +0530 Subject: [PATCH 07/28] refactor: chained callback --- .../transaction_deletion_record.py | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py index 96e5bf97891..9911d44fc3c 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py @@ -75,22 +75,37 @@ class TransactionDeletionRecord(Document): def on_cancel(self): self.db_set("status", "Cancelled") + def chain_callback(self, method): + frappe.enqueue( + "frappe.utils.background_jobs.run_doc_method", + doctype=self.doctype, + name=self.name, + doc_method=method, + queue="long", + enqueue_after_commit=True, + ) + @frappe.whitelist() def start_deletion_process(self): self.delete_bins() self.delete_lead_addresses() self.reset_company_values() + self.delete_notifications() + self.initialize_doctypes_to_be_deleted_table() + self.delete_company_transactions() + + def delete_notifications(self): if not self.clear_notifications: clear_notifications() self.db_set("clear_notifications", 1) - self.initialize_doctypes_to_be_deleted_table() - self.delete_company_transactions() + self.chain_callback("initialize_doctypes_to_be_deleted_table") def populate_doctypes_to_be_ignored_table(self): doctypes_to_be_ignored_list = get_doctypes_to_be_ignored() for doctype in doctypes_to_be_ignored_list: self.append("doctypes_to_be_ignored", {"doctype_name": doctype}) + @frappe.whitelist() def delete_bins(self): if not self.delete_bin_data: frappe.db.sql( @@ -99,6 +114,7 @@ class TransactionDeletionRecord(Document): self.company, ) self.db_set("delete_bin_data", 1) + self.chain_callback(method="delete_lead_addresses") def delete_lead_addresses(self): """Delete addresses to which leads are linked""" @@ -139,6 +155,7 @@ class TransactionDeletionRecord(Document): ) ) self.db_set("delete_leads_and_addresses", 1) + self.chain_callback(method="reset_company_values") def reset_company_values(self): if not self.reset_company_default_values: @@ -147,6 +164,7 @@ class TransactionDeletionRecord(Document): company_obj.sales_monthly_history = None company_obj.save() self.db_set("reset_company_default_values", 1) + self.chain_callback(method="delete_notifications") def initialize_doctypes_to_be_deleted_table(self): if not self.initialize_doctypes_table: @@ -162,6 +180,7 @@ class TransactionDeletionRecord(Document): # Initialize self.populate_doctypes_table(tables, docfield["parent"], docfield["fieldname"], 0) self.db_set("initialize_doctypes_table", 1) + self.chain_callback(method="delete_company_transactions") def delete_company_transactions(self): if not self.delete_transactions: @@ -194,6 +213,7 @@ class TransactionDeletionRecord(Document): if "#" in naming_series: self.update_naming_series(naming_series, docfield.doctype_name) + self.chain_callback(method="delete_company_transactions") else: frappe.db.set_value(docfield.doctype, docfield.name, "done", 1) self.db_set("delete_transactions", 1) From 49d3bcbc8df8fe9457cce55793896146705b50e5 Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Sun, 4 Feb 2024 10:57:34 +0530 Subject: [PATCH 08/28] refactor: use separate child table for summary --- .../__init__.py | 0 .../transaction_deletion_record_details.json | 59 +++++++++++++++++++ .../transaction_deletion_record_details.py | 26 ++++++++ .../transaction_deletion_record.js | 12 ++++ .../transaction_deletion_record.json | 4 +- .../transaction_deletion_record.py | 43 ++++++++------ .../transaction_deletion_record_item.json | 25 +------- .../transaction_deletion_record_item.py | 3 - 8 files changed, 126 insertions(+), 46 deletions(-) create mode 100644 erpnext/accounts/doctype/transaction_deletion_record_details/__init__.py create mode 100644 erpnext/accounts/doctype/transaction_deletion_record_details/transaction_deletion_record_details.json create mode 100644 erpnext/accounts/doctype/transaction_deletion_record_details/transaction_deletion_record_details.py diff --git a/erpnext/accounts/doctype/transaction_deletion_record_details/__init__.py b/erpnext/accounts/doctype/transaction_deletion_record_details/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/erpnext/accounts/doctype/transaction_deletion_record_details/transaction_deletion_record_details.json b/erpnext/accounts/doctype/transaction_deletion_record_details/transaction_deletion_record_details.json new file mode 100644 index 00000000000..e8a5eb6c432 --- /dev/null +++ b/erpnext/accounts/doctype/transaction_deletion_record_details/transaction_deletion_record_details.json @@ -0,0 +1,59 @@ +{ + "actions": [], + "allow_rename": 1, + "creation": "2024-02-04 10:53:32.307930", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "doctype_name", + "docfield_name", + "no_of_docs", + "done" + ], + "fields": [ + { + "fieldname": "doctype_name", + "fieldtype": "Link", + "in_list_view": 1, + "label": "DocType", + "options": "DocType", + "read_only": 1, + "reqd": 1 + }, + { + "fieldname": "docfield_name", + "fieldtype": "Data", + "in_list_view": 1, + "label": "DocField", + "read_only": 1 + }, + { + "fieldname": "no_of_docs", + "fieldtype": "Int", + "in_list_view": 1, + "label": "No of Docs", + "read_only": 1 + }, + { + "default": "0", + "fieldname": "done", + "fieldtype": "Check", + "in_list_view": 1, + "label": "Done", + "read_only": 1 + } + ], + "index_web_pages_for_search": 1, + "istable": 1, + "links": [], + "modified": "2024-02-04 10:55:52.060417", + "modified_by": "Administrator", + "module": "Accounts", + "name": "Transaction Deletion Record Details", + "owner": "Administrator", + "permissions": [], + "sort_field": "modified", + "sort_order": "DESC", + "states": [] +} \ No newline at end of file diff --git a/erpnext/accounts/doctype/transaction_deletion_record_details/transaction_deletion_record_details.py b/erpnext/accounts/doctype/transaction_deletion_record_details/transaction_deletion_record_details.py new file mode 100644 index 00000000000..bc5b5c41fdd --- /dev/null +++ b/erpnext/accounts/doctype/transaction_deletion_record_details/transaction_deletion_record_details.py @@ -0,0 +1,26 @@ +# Copyright (c) 2024, Frappe Technologies Pvt. Ltd. and contributors +# For license information, please see license.txt + +# import frappe +from frappe.model.document import Document + + +class TransactionDeletionRecordDetails(Document): + # begin: auto-generated types + # This code is auto-generated. Do not modify anything in this block. + + from typing import TYPE_CHECKING + + if TYPE_CHECKING: + from frappe.types import DF + + docfield_name: DF.Data | None + doctype_name: DF.Link + done: DF.Check + no_of_docs: DF.Int + parent: DF.Data + parentfield: DF.Data + parenttype: DF.Data + # end: auto-generated types + + pass diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.js b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.js index 1a8b52f46bd..c6bb3781bfc 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.js +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.js @@ -41,6 +41,18 @@ frappe.ui.form.on("Transaction Deletion Record", { }); }); } + + if (frm.doc.docstatus==1 && ['Queued', 'Failed'].find(x => x == frm.doc.status)) { + let execute_btn = __("Start Chain of Events") + + frm.add_custom_button(execute_btn, () => { + frm.call({ + method: 'delete_bins', + doc: frm.doc + }); + }); + } + }, }); diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json index dc35fe59553..bbc571a0816 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json @@ -34,7 +34,7 @@ "fieldname": "doctypes", "fieldtype": "Table", "label": "Summary", - "options": "Transaction Deletion Record Item", + "options": "Transaction Deletion Record Details", "read_only": 1 }, { @@ -123,7 +123,7 @@ "index_web_pages_for_search": 1, "is_submittable": 1, "links": [], - "modified": "2024-02-04 08:09:26.784109", + "modified": "2024-02-04 10:55:09.430373", "modified_by": "Administrator", "module": "Setup", "name": "Transaction Deletion Record", diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py index 9911d44fc3c..60976aa572b 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py @@ -18,6 +18,9 @@ class TransactionDeletionRecord(Document): if TYPE_CHECKING: from frappe.types import DF + from erpnext.accounts.doctype.transaction_deletion_record_details.transaction_deletion_record_details import ( + TransactionDeletionRecordDetails, + ) from erpnext.setup.doctype.transaction_deletion_record_item.transaction_deletion_record_item import ( TransactionDeletionRecordItem, ) @@ -28,7 +31,7 @@ class TransactionDeletionRecord(Document): delete_bin_data: DF.Check delete_leads_and_addresses: DF.Check delete_transactions: DF.Check - doctypes: DF.Table[TransactionDeletionRecordItem] + doctypes: DF.Table[TransactionDeletionRecordDetails] doctypes_to_be_ignored: DF.Table[TransactionDeletionRecordItem] initialize_doctypes_table: DF.Check reset_company_default_values: DF.Check @@ -37,7 +40,7 @@ class TransactionDeletionRecord(Document): def __init__(self, *args, **kwargs): super(TransactionDeletionRecord, self).__init__(*args, **kwargs) - self.batch_size = 5000 + self.batch_size = 5 def validate(self): frappe.only_for("System Manager") @@ -75,7 +78,7 @@ class TransactionDeletionRecord(Document): def on_cancel(self): self.db_set("status", "Cancelled") - def chain_callback(self, method): + def chain_call(self, method): frappe.enqueue( "frappe.utils.background_jobs.run_doc_method", doctype=self.doctype, @@ -98,7 +101,7 @@ class TransactionDeletionRecord(Document): if not self.clear_notifications: clear_notifications() self.db_set("clear_notifications", 1) - self.chain_callback("initialize_doctypes_to_be_deleted_table") + self.chain_call("initialize_doctypes_to_be_deleted_table") def populate_doctypes_to_be_ignored_table(self): doctypes_to_be_ignored_list = get_doctypes_to_be_ignored() @@ -114,7 +117,7 @@ class TransactionDeletionRecord(Document): self.company, ) self.db_set("delete_bin_data", 1) - self.chain_callback(method="delete_lead_addresses") + self.chain_call(method="delete_lead_addresses") def delete_lead_addresses(self): """Delete addresses to which leads are linked""" @@ -155,7 +158,7 @@ class TransactionDeletionRecord(Document): ) ) self.db_set("delete_leads_and_addresses", 1) - self.chain_callback(method="reset_company_values") + self.chain_call(method="reset_company_values") def reset_company_values(self): if not self.reset_company_default_values: @@ -164,7 +167,7 @@ class TransactionDeletionRecord(Document): company_obj.sales_monthly_history = None company_obj.save() self.db_set("reset_company_default_values", 1) - self.chain_callback(method="delete_notifications") + self.chain_call(method="delete_notifications") def initialize_doctypes_to_be_deleted_table(self): if not self.initialize_doctypes_table: @@ -180,7 +183,7 @@ class TransactionDeletionRecord(Document): # Initialize self.populate_doctypes_table(tables, docfield["parent"], docfield["fieldname"], 0) self.db_set("initialize_doctypes_table", 1) - self.chain_callback(method="delete_company_transactions") + self.chain_call(method="delete_company_transactions") def delete_company_transactions(self): if not self.delete_transactions: @@ -203,20 +206,24 @@ class TransactionDeletionRecord(Document): self.delete_communications(docfield.doctype_name, reference_doc_names) self.delete_comments(docfield.doctype_name, reference_doc_names) self.unlink_attachments(docfield.doctype_name, reference_doc_names) - self.delete_child_tables(docfield.doctype_name, reference_doc_names) - self.delete_docs_linked_with_specified_company(docfield.doctype_name, docfield.docfield_name) - + self.delete_docs_linked_with_specified_company(docfield.doctype_name, reference_doc_names) + processed = int(docfield.no_of_docs) + len(reference_doc_names) + frappe.db.set_value(docfield.doctype, docfield.name, "no_of_docs", processed) + else: naming_series = frappe.db.get_value("DocType", docfield.doctype_name, "autoname") - # TODO: do this at the end of each doctype if naming_series: if "#" in naming_series: self.update_naming_series(naming_series, docfield.doctype_name) - - self.chain_callback(method="delete_company_transactions") - else: frappe.db.set_value(docfield.doctype, docfield.name, "done", 1) - self.db_set("delete_transactions", 1) + + pending_doctypes = frappe.db.get_all( + docfield.doctype, filters={"parent": self.name, "done": 0}, pluck="doctype_name" + ) + if pending_doctypes: + self.chain_call(method="delete_company_transactions") + else: + self.db_set("delete_transactions", 1) def get_doctypes_to_be_ignored_list(self): singles = frappe.get_all("DocType", filters={"issingle": 1}, pluck="name") @@ -261,8 +268,8 @@ class TransactionDeletionRecord(Document): for table in child_tables: frappe.db.delete(table, {"parent": ["in", reference_doc_names]}) - def delete_docs_linked_with_specified_company(self, doctype, company_fieldname): - frappe.db.delete(doctype, {company_fieldname: self.company}) + def delete_docs_linked_with_specified_company(self, doctype, reference_doc_names): + frappe.db.delete(doctype, {"name": ("in", reference_doc_names)}) def update_naming_series(self, naming_series, doctype_name): if "." in naming_series: diff --git a/erpnext/setup/doctype/transaction_deletion_record_item/transaction_deletion_record_item.json b/erpnext/setup/doctype/transaction_deletion_record_item/transaction_deletion_record_item.json index 4e5e1846999..89db63694c2 100644 --- a/erpnext/setup/doctype/transaction_deletion_record_item/transaction_deletion_record_item.json +++ b/erpnext/setup/doctype/transaction_deletion_record_item/transaction_deletion_record_item.json @@ -5,10 +5,7 @@ "editable_grid": 1, "engine": "InnoDB", "field_order": [ - "doctype_name", - "docfield_name", - "no_of_docs", - "done" + "doctype_name" ], "fields": [ { @@ -18,30 +15,12 @@ "label": "DocType", "options": "DocType", "reqd": 1 - }, - { - "fieldname": "no_of_docs", - "fieldtype": "Data", - "in_list_view": 1, - "label": "Number of Docs" - }, - { - "default": "0", - "fieldname": "done", - "fieldtype": "Check", - "in_list_view": 1, - "label": "Done" - }, - { - "fieldname": "docfield_name", - "fieldtype": "Data", - "label": "DocField Name" } ], "index_web_pages_for_search": 1, "istable": 1, "links": [], - "modified": "2024-02-03 21:06:32.274445", + "modified": "2024-02-04 10:56:27.413691", "modified_by": "Administrator", "module": "Setup", "name": "Transaction Deletion Record Item", diff --git a/erpnext/setup/doctype/transaction_deletion_record_item/transaction_deletion_record_item.py b/erpnext/setup/doctype/transaction_deletion_record_item/transaction_deletion_record_item.py index ce716ac477c..906660739e4 100644 --- a/erpnext/setup/doctype/transaction_deletion_record_item/transaction_deletion_record_item.py +++ b/erpnext/setup/doctype/transaction_deletion_record_item/transaction_deletion_record_item.py @@ -15,10 +15,7 @@ class TransactionDeletionRecordItem(Document): if TYPE_CHECKING: from frappe.types import DF - docfield_name: DF.Data | None doctype_name: DF.Link - done: DF.Check - no_of_docs: DF.Data | None parent: DF.Data parentfield: DF.Data parenttype: DF.Data From b98a5e4edcd896b05e4e7fd5874c56ceb9d95bf0 Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Sun, 4 Feb 2024 15:26:33 +0530 Subject: [PATCH 09/28] chore: remove unwanted UI code --- .../transaction_deletion_record.js | 29 +++---------------- 1 file changed, 4 insertions(+), 25 deletions(-) diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.js b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.js index c6bb3781bfc..027bbcb4b20 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.js +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.js @@ -10,40 +10,19 @@ frappe.ui.form.on("Transaction Deletion Record", { callback: function (r) { doctypes_to_be_ignored_array = r.message; populate_doctypes_to_be_ignored(doctypes_to_be_ignored_array, frm); - frm.fields_dict["doctypes_to_be_ignored"].grid.set_column_disp("no_of_docs", false); - frm.refresh_field("doctypes_to_be_ignored"); - }, + frm.refresh_field('doctypes_to_be_ignored'); + } }); } frm.get_field('doctypes_to_be_ignored').grid.cannot_add_rows = true; - frm.fields_dict['doctypes_to_be_ignored'].grid.set_column_disp('no_of_docs', false); - frm.fields_dict['doctypes_to_be_ignored'].grid.set_column_disp('done', false); - frm.refresh_field('doctypes_to_be_ignored'); - frm.get_field('doctypes').grid.cannot_add_rows = true; - frm.fields_dict['doctypes'].grid.set_column_disp('no_of_docs', true); - frm.refresh_field('doctypes'); }, - refresh: function (frm) { - frm.fields_dict["doctypes_to_be_ignored"].grid.set_column_disp("no_of_docs", false); - frm.refresh_field("doctypes_to_be_ignored"); - + refresh: function(frm) { if (frm.doc.docstatus==1 && ['Queued', 'Failed'].find(x => x == frm.doc.status)) { - let execute_btn = __("Start / Resume") - - frm.add_custom_button(execute_btn, () => { - frm.call({ - method: 'start_deletion_process', - doc: frm.doc - }); - }); - } - - if (frm.doc.docstatus==1 && ['Queued', 'Failed'].find(x => x == frm.doc.status)) { - let execute_btn = __("Start Chain of Events") + let execute_btn = __("Start Deletion") frm.add_custom_button(execute_btn, () => { frm.call({ From 7c4cff2649daa52d6cabf8a046b3867ab60e0da6 Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Sun, 4 Feb 2024 15:29:39 +0530 Subject: [PATCH 10/28] refactor: make Excluded doctype table read only --- .../transaction_deletion_record.js | 4 +--- .../transaction_deletion_record.json | 5 +++-- .../transaction_deletion_record.py | 1 + 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.js b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.js index 027bbcb4b20..ed70ebb5f70 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.js +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.js @@ -15,9 +15,6 @@ frappe.ui.form.on("Transaction Deletion Record", { }); } - - frm.get_field('doctypes_to_be_ignored').grid.cannot_add_rows = true; - }, refresh: function(frm) { @@ -25,6 +22,7 @@ frappe.ui.form.on("Transaction Deletion Record", { let execute_btn = __("Start Deletion") frm.add_custom_button(execute_btn, () => { + // Entry point for chain of events frm.call({ method: 'delete_bins', doc: frm.doc diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json index bbc571a0816..bd45b1c109d 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json @@ -41,7 +41,8 @@ "fieldname": "doctypes_to_be_ignored", "fieldtype": "Table", "label": "Excluded DocTypes", - "options": "Transaction Deletion Record Item" + "options": "Transaction Deletion Record Item", + "read_only": 1 }, { "fieldname": "amended_from", @@ -123,7 +124,7 @@ "index_web_pages_for_search": 1, "is_submittable": 1, "links": [], - "modified": "2024-02-04 10:55:09.430373", + "modified": "2024-02-04 15:28:29.532826", "modified_by": "Administrator", "module": "Setup", "name": "Transaction Deletion Record", diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py index 60976aa572b..0d553ca19e1 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py @@ -110,6 +110,7 @@ class TransactionDeletionRecord(Document): @frappe.whitelist() def delete_bins(self): + # This methid is the entry point for the chain of events that follow if not self.delete_bin_data: frappe.db.sql( """delete from `tabBin` where warehouse in From 86b5e2e2779eb498a8525ca07c6e25335e6f2f5d Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Sun, 4 Feb 2024 15:37:21 +0530 Subject: [PATCH 11/28] refactor: validate status before running events --- .../transaction_deletion_record.py | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py index 0d553ca19e1..63b28017685 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py @@ -6,7 +6,7 @@ import frappe from frappe import _, qb from frappe.desk.notifications import clear_notifications from frappe.model.document import Document -from frappe.utils import cint, create_batch +from frappe.utils import cint, create_batch, get_link_to_form class TransactionDeletionRecord(Document): @@ -40,7 +40,7 @@ class TransactionDeletionRecord(Document): def __init__(self, *args, **kwargs): super(TransactionDeletionRecord, self).__init__(*args, **kwargs) - self.batch_size = 5 + self.batch_size = 5000 def validate(self): frappe.only_for("System Manager") @@ -98,6 +98,7 @@ class TransactionDeletionRecord(Document): self.delete_company_transactions() def delete_notifications(self): + self.validate_doc_status() if not self.clear_notifications: clear_notifications() self.db_set("clear_notifications", 1) @@ -108,9 +109,19 @@ class TransactionDeletionRecord(Document): for doctype in doctypes_to_be_ignored_list: self.append("doctypes_to_be_ignored", {"doctype_name": doctype}) + def validate_doc_status(self): + if self.status != "Running": + frappe.throw( + _("{0} is not running. Cannot trigger events for this Document").format( + get_link_to_form("Transaction Deletion Record", self.name) + ) + ) + @frappe.whitelist() def delete_bins(self): # This methid is the entry point for the chain of events that follow + self.db_set("status", "Running") + if not self.delete_bin_data: frappe.db.sql( """delete from `tabBin` where warehouse in @@ -122,6 +133,7 @@ class TransactionDeletionRecord(Document): def delete_lead_addresses(self): """Delete addresses to which leads are linked""" + self.validate_doc_status() if not self.delete_leads_and_addresses: leads = frappe.get_all("Lead", filters={"company": self.company}) leads = ["'%s'" % row.get("name") for row in leads] @@ -162,6 +174,7 @@ class TransactionDeletionRecord(Document): self.chain_call(method="reset_company_values") def reset_company_values(self): + self.validate_doc_status() if not self.reset_company_default_values: company_obj = frappe.get_doc("Company", self.company) company_obj.total_monthly_sales = 0 @@ -171,6 +184,7 @@ class TransactionDeletionRecord(Document): self.chain_call(method="delete_notifications") def initialize_doctypes_to_be_deleted_table(self): + self.validate_doc_status() if not self.initialize_doctypes_table: doctypes_to_be_ignored_list = self.get_doctypes_to_be_ignored_list() docfields = self.get_doctypes_with_company_field(doctypes_to_be_ignored_list) @@ -187,6 +201,7 @@ class TransactionDeletionRecord(Document): self.chain_call(method="delete_company_transactions") def delete_company_transactions(self): + self.validate_doc_status() if not self.delete_transactions: doctypes_to_be_ignored_list = self.get_doctypes_to_be_ignored_list() docfields = self.get_doctypes_with_company_field(doctypes_to_be_ignored_list) @@ -212,6 +227,7 @@ class TransactionDeletionRecord(Document): processed = int(docfield.no_of_docs) + len(reference_doc_names) frappe.db.set_value(docfield.doctype, docfield.name, "no_of_docs", processed) else: + # reset naming series naming_series = frappe.db.get_value("DocType", docfield.doctype_name, "autoname") if naming_series: if "#" in naming_series: @@ -224,6 +240,7 @@ class TransactionDeletionRecord(Document): if pending_doctypes: self.chain_call(method="delete_company_transactions") else: + self.db_set("status", "Completed") self.db_set("delete_transactions", 1) def get_doctypes_to_be_ignored_list(self): From 1014940953ed54f31a170e0e99d03c1c5f1cd022 Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Sun, 4 Feb 2024 16:11:42 +0530 Subject: [PATCH 12/28] chore: show correct status in list view --- .../transaction_deletion_record_list.js | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record_list.js b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record_list.js index 08a35df2c17..7c7b8ff25a7 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record_list.js +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record_list.js @@ -1,12 +1,16 @@ // Copyright (c) 2018, Frappe Technologies Pvt. Ltd. and Contributors // License: GNU General Public License v3. See license.txt -frappe.listview_settings["Transaction Deletion Record"] = { - get_indicator: function (doc) { - if (doc.docstatus == 0) { - return [__("Draft"), "red"]; - } else { - return [__("Completed"), "green"]; - } +frappe.listview_settings['Transaction Deletion Record'] = { + add_fields: ["status"], + get_indicator: function(doc) { + let colors = { + 'Queued': 'orange', + 'Completed': 'green', + 'Running': 'blue', + 'Failed': 'red', + }; + let status = doc.status; + return [__(status), colors[status], 'status,=,'+status]; }, }; From 2dbe68a09d0bf9a13ef0d3bb8f071a7634291766 Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Mon, 5 Feb 2024 10:21:12 +0530 Subject: [PATCH 13/28] refactor: reset all flags and remove unwanted code --- .../transaction_deletion_record.json | 4 +++- .../transaction_deletion_record.py | 19 ++++++++++--------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json index bd45b1c109d..aa06d14b170 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json @@ -57,6 +57,7 @@ "fieldname": "status", "fieldtype": "Select", "label": "Status", + "no_copy": 1, "options": "Queued\nRunning\nFailed\nCompleted\nCancelled", "read_only": 1 }, @@ -118,13 +119,14 @@ "fieldname": "initialize_doctypes_table", "fieldtype": "Check", "label": "Initialize Summary Table", + "no_copy": 1, "read_only": 1 } ], "index_web_pages_for_search": 1, "is_submittable": 1, "links": [], - "modified": "2024-02-04 15:28:29.532826", + "modified": "2024-02-05 10:25:28.462255", "modified_by": "Administrator", "module": "Setup", "name": "Transaction Deletion Record", diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py index 63b28017685..5f430b85f58 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py @@ -58,6 +58,15 @@ class TransactionDeletionRecord(Document): ) def before_submit(self): + if queued_docs := frappe.db.get_all( + "Transaction Deletion Record", filters={"company": self.company, "status": "Queued"} + ): + frappe.throw( + _("There is another document: {0} Queued. Cannot queue multi docs for one company.").format( + self.queued_docs + ) + ) + if not self.doctypes_to_be_ignored: self.populate_doctypes_to_be_ignored_table() @@ -66,6 +75,7 @@ class TransactionDeletionRecord(Document): self.delete_bin_data = 0 self.delete_leads_and_addresses = 0 self.delete_transactions = 0 + self.initialize_doctypes_table = 0 self.reset_company_default_values = 0 def before_save(self): @@ -88,15 +98,6 @@ class TransactionDeletionRecord(Document): enqueue_after_commit=True, ) - @frappe.whitelist() - def start_deletion_process(self): - self.delete_bins() - self.delete_lead_addresses() - self.reset_company_values() - self.delete_notifications() - self.initialize_doctypes_to_be_deleted_table() - self.delete_company_transactions() - def delete_notifications(self): self.validate_doc_status() if not self.clear_notifications: From 55e93b3fe14b492e2f648b6ee860e902d87013e6 Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Mon, 5 Feb 2024 10:42:59 +0530 Subject: [PATCH 14/28] refactor: no copy on summary table and more validations --- .../transaction_deletion_record.json | 3 ++- .../transaction_deletion_record.py | 14 ++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json index aa06d14b170..6e057ace4a6 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json @@ -34,6 +34,7 @@ "fieldname": "doctypes", "fieldtype": "Table", "label": "Summary", + "no_copy": 1, "options": "Transaction Deletion Record Details", "read_only": 1 }, @@ -126,7 +127,7 @@ "index_web_pages_for_search": 1, "is_submittable": 1, "links": [], - "modified": "2024-02-05 10:25:28.462255", + "modified": "2024-02-05 10:36:34.229864", "modified_by": "Administrator", "module": "Setup", "name": "Transaction Deletion Record", diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py index 5f430b85f58..c2c173e0514 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py @@ -6,7 +6,7 @@ import frappe from frappe import _, qb from frappe.desk.notifications import clear_notifications from frappe.model.document import Document -from frappe.utils import cint, create_batch, get_link_to_form +from frappe.utils import cint, comma_and, create_batch, get_link_to_form class TransactionDeletionRecord(Document): @@ -59,11 +59,16 @@ class TransactionDeletionRecord(Document): def before_submit(self): if queued_docs := frappe.db.get_all( - "Transaction Deletion Record", filters={"company": self.company, "status": "Queued"} + "Transaction Deletion Record", + filters={"company": self.company, "status": ("in", ["Running", "Queued"]), "docstatus": 1}, + pluck="name", ): frappe.throw( - _("There is another document: {0} Queued. Cannot queue multi docs for one company.").format( - self.queued_docs + _( + "Cannot queue multi docs for one company. {0} is already queued/running for company: {1}" + ).format( + comma_and([get_link_to_form("Transaction Deletion Record", x) for x in queued_docs]), + frappe.bold(self.company), ) ) @@ -80,6 +85,7 @@ class TransactionDeletionRecord(Document): def before_save(self): self.status = "" + self.doctypes.clear() self.reset_task_flags() def on_submit(self): From 31a2da552b0773f8b7caf510fa06e54cdc1f2e9c Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Mon, 5 Feb 2024 11:52:00 +0530 Subject: [PATCH 15/28] refactor: validations to prevent duplicate jobs --- .../transaction_deletion_record.py | 75 +++++++++++++++---- 1 file changed, 59 insertions(+), 16 deletions(-) diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py index c2c173e0514..71892bbba1a 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py @@ -7,6 +7,7 @@ from frappe import _, qb from frappe.desk.notifications import clear_notifications from frappe.model.document import Document from frappe.utils import cint, comma_and, create_batch, get_link_to_form +from frappe.utils.background_jobs import get_job, is_job_enqueued class TransactionDeletionRecord(Document): @@ -41,6 +42,15 @@ class TransactionDeletionRecord(Document): def __init__(self, *args, **kwargs): super(TransactionDeletionRecord, self).__init__(*args, **kwargs) self.batch_size = 5000 + # Tasks are listged by their execution order + self.task_to_internal_method_map = { + "Delete Bins": "delete_bins", + "Delete Leads and Addresses": "delete_lead_addresses", + "Reset Company Values": "reset_company_values", + "Clear Notifications": "delete_notifications", + "Initialize Summary Table": "initialize_doctypes_to_be_deleted_table", + "Delete Transactions": "delete_company_transactions", + } def validate(self): frappe.only_for("System Manager") @@ -57,6 +67,16 @@ class TransactionDeletionRecord(Document): title=_("Not Allowed"), ) + def generate_job_name_for_task(self, task=None): + method = self.task_to_internal_method_map[task] + return f"{self.name}_{method}" + + def generate_job_name_for_all_tasks(self): + job_names = [] + for method in self.task_to_internal_method_map.values(): + job_names.append(self.generate_job_name_for_task) + return job_names + def before_submit(self): if queued_docs := frappe.db.get_all( "Transaction Deletion Record", @@ -65,7 +85,7 @@ class TransactionDeletionRecord(Document): ): frappe.throw( _( - "Cannot queue multi docs for one company. {0} is already queued/running for company: {1}" + "Cannot enqueue multi docs for one company. {0} is already queued/running for company: {1}" ).format( comma_and([get_link_to_form("Transaction Deletion Record", x) for x in queued_docs]), frappe.bold(self.company), @@ -94,28 +114,47 @@ class TransactionDeletionRecord(Document): def on_cancel(self): self.db_set("status", "Cancelled") - def chain_call(self, method): - frappe.enqueue( - "frappe.utils.background_jobs.run_doc_method", - doctype=self.doctype, - name=self.name, - doc_method=method, - queue="long", - enqueue_after_commit=True, - ) + def chain_call(self, task=None): + if task and task in self.task_to_internal_method_map: + method = self.task_to_internal_method_map[task] + job_id = self.generate_job_name_for_task(task) + + frappe.enqueue( + "frappe.utils.background_jobs.run_doc_method", + doctype=self.doctype, + name=self.name, + doc_method=method, + job_id=job_id, + queue="long", + enqueue_after_commit=True, + ) def delete_notifications(self): self.validate_doc_status() if not self.clear_notifications: clear_notifications() self.db_set("clear_notifications", 1) - self.chain_call("initialize_doctypes_to_be_deleted_table") + self.chain_call(task="Initialize Summary Table") def populate_doctypes_to_be_ignored_table(self): doctypes_to_be_ignored_list = get_doctypes_to_be_ignored() for doctype in doctypes_to_be_ignored_list: self.append("doctypes_to_be_ignored", {"doctype_name": doctype}) + def validate_running_task_for_doc(self, job_names: list = None): + # at most only one task should be runnning + running_tasks = [] + for x in job_names: + if is_job_enqueued(x): + running_tasks.append(get_job(x).get_id()) + + if running_tasks: + frappe.throw( + _("{0} is already running for {1}").format( + comma_and([get_link_to_form("RQ Job", x) for x in running_tasks]), self.name + ) + ) + def validate_doc_status(self): if self.status != "Running": frappe.throw( @@ -123,6 +162,9 @@ class TransactionDeletionRecord(Document): get_link_to_form("Transaction Deletion Record", self.name) ) ) + # make sure that job none of tasks are already running + job_names = self.generate_job_name_for_all_tasks() + self.validate_running_task_for_doc(job_names=job_names) @frappe.whitelist() def delete_bins(self): @@ -136,7 +178,7 @@ class TransactionDeletionRecord(Document): self.company, ) self.db_set("delete_bin_data", 1) - self.chain_call(method="delete_lead_addresses") + self.chain_call(task="Delete Leads and Addresses") def delete_lead_addresses(self): """Delete addresses to which leads are linked""" @@ -178,7 +220,7 @@ class TransactionDeletionRecord(Document): ) ) self.db_set("delete_leads_and_addresses", 1) - self.chain_call(method="reset_company_values") + self.chain_call(task="Reset Company Values") def reset_company_values(self): self.validate_doc_status() @@ -188,7 +230,7 @@ class TransactionDeletionRecord(Document): company_obj.sales_monthly_history = None company_obj.save() self.db_set("reset_company_default_values", 1) - self.chain_call(method="delete_notifications") + self.chain_call(task="Clear Notifications") def initialize_doctypes_to_be_deleted_table(self): self.validate_doc_status() @@ -205,7 +247,7 @@ class TransactionDeletionRecord(Document): # Initialize self.populate_doctypes_table(tables, docfield["parent"], docfield["fieldname"], 0) self.db_set("initialize_doctypes_table", 1) - self.chain_call(method="delete_company_transactions") + self.chain_call(task="Delete Transactions") def delete_company_transactions(self): self.validate_doc_status() @@ -245,7 +287,8 @@ class TransactionDeletionRecord(Document): docfield.doctype, filters={"parent": self.name, "done": 0}, pluck="doctype_name" ) if pending_doctypes: - self.chain_call(method="delete_company_transactions") + # as method is enqueued after commit, calling itself will not make validate_doc_status to throw + self.chain_call(task="Delete Transactions") else: self.db_set("status", "Completed") self.db_set("delete_transactions", 1) From 98afb4d468145ac2270362841468f2ccb05b501f Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Mon, 5 Feb 2024 17:35:14 +0530 Subject: [PATCH 16/28] chore: hide docfield in list view --- .../transaction_deletion_record_details.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/erpnext/accounts/doctype/transaction_deletion_record_details/transaction_deletion_record_details.json b/erpnext/accounts/doctype/transaction_deletion_record_details/transaction_deletion_record_details.json index e8a5eb6c432..fe4b0852ac1 100644 --- a/erpnext/accounts/doctype/transaction_deletion_record_details/transaction_deletion_record_details.json +++ b/erpnext/accounts/doctype/transaction_deletion_record_details/transaction_deletion_record_details.json @@ -24,7 +24,6 @@ { "fieldname": "docfield_name", "fieldtype": "Data", - "in_list_view": 1, "label": "DocField", "read_only": 1 }, @@ -47,7 +46,7 @@ "index_web_pages_for_search": 1, "istable": 1, "links": [], - "modified": "2024-02-04 10:55:52.060417", + "modified": "2024-02-05 17:35:09.556054", "modified_by": "Administrator", "module": "Accounts", "name": "Transaction Deletion Record Details", From 78c9cc63b1011761dd3fed84edb5c0b41ff9f3f5 Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Mon, 5 Feb 2024 20:35:29 +0530 Subject: [PATCH 17/28] refactor: make sure only one task is running for doc --- .../transaction_deletion_record.js | 2 +- .../transaction_deletion_record.py | 48 ++++++++++++------- 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.js b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.js index ed70ebb5f70..ccf09a6c38b 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.js +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.js @@ -24,7 +24,7 @@ frappe.ui.form.on("Transaction Deletion Record", { frm.add_custom_button(execute_btn, () => { // Entry point for chain of events frm.call({ - method: 'delete_bins', + method: 'process_tasks', doc: frm.doc }); }); diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py index 71892bbba1a..be161e30322 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py @@ -1,6 +1,7 @@ # Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and contributors # For license information, please see license.txt +from collections import OrderedDict import frappe from frappe import _, qb @@ -42,15 +43,17 @@ class TransactionDeletionRecord(Document): def __init__(self, *args, **kwargs): super(TransactionDeletionRecord, self).__init__(*args, **kwargs) self.batch_size = 5000 - # Tasks are listged by their execution order - self.task_to_internal_method_map = { - "Delete Bins": "delete_bins", - "Delete Leads and Addresses": "delete_lead_addresses", - "Reset Company Values": "reset_company_values", - "Clear Notifications": "delete_notifications", - "Initialize Summary Table": "initialize_doctypes_to_be_deleted_table", - "Delete Transactions": "delete_company_transactions", - } + # Tasks are listed by their execution order + self.task_to_internal_method_map = OrderedDict( + { + "Delete Bins": "delete_bins", + "Delete Leads and Addresses": "delete_lead_addresses", + "Reset Company Values": "reset_company_values", + "Clear Notifications": "delete_notifications", + "Initialize Summary Table": "initialize_doctypes_to_be_deleted_table", + "Delete Transactions": "delete_company_transactions", + } + ) def validate(self): frappe.only_for("System Manager") @@ -71,10 +74,19 @@ class TransactionDeletionRecord(Document): method = self.task_to_internal_method_map[task] return f"{self.name}_{method}" + def generate_job_name_for_next_tasks(self, task=None): + job_names = [] + current_task_idx = list(self.task_to_internal_method_map).index(task) + for idx, task in enumerate(self.task_to_internal_method_map.keys(), 0): + # generate job_name for next tasks + if idx > current_task_idx: + job_names.append(self.generate_job_name_for_task(task)) + return job_names + def generate_job_name_for_all_tasks(self): job_names = [] - for method in self.task_to_internal_method_map.values(): - job_names.append(self.generate_job_name_for_task) + for task in self.task_to_internal_method_map.keys(): + job_names.append(self.generate_job_name_for_task(task)) return job_names def before_submit(self): @@ -116,6 +128,10 @@ class TransactionDeletionRecord(Document): def chain_call(self, task=None): if task and task in self.task_to_internal_method_map: + # make sure that none of next tasks are already running + job_names = self.generate_job_name_for_next_tasks(task=task) + self.validate_running_task_for_doc(job_names=job_names) + method = self.task_to_internal_method_map[task] job_id = self.generate_job_name_for_task(task) @@ -162,15 +178,15 @@ class TransactionDeletionRecord(Document): get_link_to_form("Transaction Deletion Record", self.name) ) ) - # make sure that job none of tasks are already running - job_names = self.generate_job_name_for_all_tasks() - self.validate_running_task_for_doc(job_names=job_names) @frappe.whitelist() - def delete_bins(self): - # This methid is the entry point for the chain of events that follow + def process_tasks(self): + # This method is the entry point for the chain of events that follow self.db_set("status", "Running") + self.chain_call(task="Delete Bins") + def delete_bins(self): + self.validate_doc_status() if not self.delete_bin_data: frappe.db.sql( """delete from `tabBin` where warehouse in From ec194ef076037b16edea68728069b3308a9a4216 Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Wed, 7 Feb 2024 20:54:29 +0530 Subject: [PATCH 18/28] refactor: barebones hook on all doctypes with 'company' field --- erpnext/hooks.py | 5 ++++- .../transaction_deletion_record.py | 21 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/erpnext/hooks.py b/erpnext/hooks.py index 308e6ca011e..492ae701d52 100644 --- a/erpnext/hooks.py +++ b/erpnext/hooks.py @@ -309,7 +309,10 @@ period_closing_doctypes = [ doc_events = { "*": { - "validate": "erpnext.support.doctype.service_level_agreement.service_level_agreement.apply", + "validate": [ + "erpnext.support.doctype.service_level_agreement.service_level_agreement.apply", + "erpnext.setup.doctype.transaction_deletion_record.transaction_deletion_record.check_for_running_deletion_job", + ], }, tuple(period_closing_doctypes): { "validate": "erpnext.accounts.doctype.accounting_period.accounting_period.validate_accounting_period_on_doc_save", diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py index be161e30322..5a49a7d1d7f 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py @@ -450,3 +450,24 @@ def get_doctypes_to_be_ignored(): doctypes_to_be_ignored.extend(frappe.get_hooks("company_data_to_be_ignored") or []) return doctypes_to_be_ignored + + +def check_for_running_deletion_job(doc, method=None): + df = qb.DocType("DocField") + if ( + not_allowed := qb.from_(df) + .select(df.parent) + .where((df.fieldname == "company") & (df.parent == doc.doctype)) + .run() + ): + if running_deletion_jobs := frappe.db.get_all( + "Transaction Deletion Record", + filters={"docstatus": 1, "company": doc.company, "status": "Running"}, + ): + frappe.throw( + _( + "Transaction Deletion job {0} is running for this Company. Cannot make any transactions until the deletion job is completed" + ).format( + get_link_to_form("Transaction Deletion Record", running_deletion_jobs[0].name) + ) + ) From 30463657bf7f44321b690a79c94e24d6286d486e Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Thu, 8 Feb 2024 16:08:59 +0530 Subject: [PATCH 19/28] refactor: better method naming --- .../transaction_deletion_record.py | 30 ++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py index 5a49a7d1d7f..170b1bfe72d 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py @@ -126,31 +126,40 @@ class TransactionDeletionRecord(Document): def on_cancel(self): self.db_set("status", "Cancelled") - def chain_call(self, task=None): + def enqueue_task(self, task: str | None = None): if task and task in self.task_to_internal_method_map: # make sure that none of next tasks are already running job_names = self.generate_job_name_for_next_tasks(task=task) self.validate_running_task_for_doc(job_names=job_names) - method = self.task_to_internal_method_map[task] + # method = self.task_to_internal_method_map[task] + # Generate Job Id to uniquely identify each task for this document job_id = self.generate_job_name_for_task(task) frappe.enqueue( "frappe.utils.background_jobs.run_doc_method", doctype=self.doctype, name=self.name, - doc_method=method, + doc_method="execute_task", job_id=job_id, queue="long", enqueue_after_commit=True, + task_to_execute=task, ) + def execute_task(self, task_to_execute: str | None = None): + if task_to_execute: + pass + method = self.task_to_internal_method_map[task_to_execute] + if task := getattr(self, method, None): + task() + def delete_notifications(self): self.validate_doc_status() if not self.clear_notifications: clear_notifications() self.db_set("clear_notifications", 1) - self.chain_call(task="Initialize Summary Table") + self.enqueue_task(task="Initialize Summary Table") def populate_doctypes_to_be_ignored_table(self): doctypes_to_be_ignored_list = get_doctypes_to_be_ignored() @@ -183,7 +192,7 @@ class TransactionDeletionRecord(Document): def process_tasks(self): # This method is the entry point for the chain of events that follow self.db_set("status", "Running") - self.chain_call(task="Delete Bins") + self.enqueue_task(task="Delete Bins") def delete_bins(self): self.validate_doc_status() @@ -194,7 +203,7 @@ class TransactionDeletionRecord(Document): self.company, ) self.db_set("delete_bin_data", 1) - self.chain_call(task="Delete Leads and Addresses") + self.enqueue_task(task="Delete Leads and Addresses") def delete_lead_addresses(self): """Delete addresses to which leads are linked""" @@ -236,7 +245,7 @@ class TransactionDeletionRecord(Document): ) ) self.db_set("delete_leads_and_addresses", 1) - self.chain_call(task="Reset Company Values") + self.enqueue_task(task="Reset Company Values") def reset_company_values(self): self.validate_doc_status() @@ -246,7 +255,7 @@ class TransactionDeletionRecord(Document): company_obj.sales_monthly_history = None company_obj.save() self.db_set("reset_company_default_values", 1) - self.chain_call(task="Clear Notifications") + self.enqueue_task(task="Clear Notifications") def initialize_doctypes_to_be_deleted_table(self): self.validate_doc_status() @@ -263,7 +272,7 @@ class TransactionDeletionRecord(Document): # Initialize self.populate_doctypes_table(tables, docfield["parent"], docfield["fieldname"], 0) self.db_set("initialize_doctypes_table", 1) - self.chain_call(task="Delete Transactions") + self.enqueue_task(task="Delete Transactions") def delete_company_transactions(self): self.validate_doc_status() @@ -304,7 +313,8 @@ class TransactionDeletionRecord(Document): ) if pending_doctypes: # as method is enqueued after commit, calling itself will not make validate_doc_status to throw - self.chain_call(task="Delete Transactions") + # recursively call this task to delete all transactions + self.enqueue_task(task="Delete Transactions") else: self.db_set("status", "Completed") self.db_set("delete_transactions", 1) From eea260b9f9c4a4a3a5a716b1d74a45c7569d4f98 Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Tue, 19 Mar 2024 11:18:11 +0530 Subject: [PATCH 20/28] chore: code cleanup --- .../transaction_deletion_record.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py index 170b1bfe72d..039d9a58413 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py @@ -132,7 +132,6 @@ class TransactionDeletionRecord(Document): job_names = self.generate_job_name_for_next_tasks(task=task) self.validate_running_task_for_doc(job_names=job_names) - # method = self.task_to_internal_method_map[task] # Generate Job Id to uniquely identify each task for this document job_id = self.generate_job_name_for_task(task) @@ -149,7 +148,6 @@ class TransactionDeletionRecord(Document): def execute_task(self, task_to_execute: str | None = None): if task_to_execute: - pass method = self.task_to_internal_method_map[task_to_execute] if task := getattr(self, method, None): task() @@ -309,7 +307,9 @@ class TransactionDeletionRecord(Document): frappe.db.set_value(docfield.doctype, docfield.name, "done", 1) pending_doctypes = frappe.db.get_all( - docfield.doctype, filters={"parent": self.name, "done": 0}, pluck="doctype_name" + "Transaction Deletion Record Details", + filters={"parent": self.name, "done": 0}, + pluck="doctype_name", ) if pending_doctypes: # as method is enqueued after commit, calling itself will not make validate_doc_status to throw From 4a55240e630fedf1590a2c324528fd5934066d37 Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Tue, 19 Mar 2024 17:39:20 +0530 Subject: [PATCH 21/28] refactor: exception propogation --- .../transaction_deletion_record.json | 9 ++++++++- .../transaction_deletion_record.py | 13 ++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json index 6e057ace4a6..8291a4ffc24 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json @@ -10,6 +10,7 @@ "column_break_txbg", "status", "tasks_section", + "error_log", "delete_bin_data", "delete_leads_and_addresses", "reset_company_default_values", @@ -122,12 +123,18 @@ "label": "Initialize Summary Table", "no_copy": 1, "read_only": 1 + }, + { + "depends_on": "eval: doc.error_log", + "fieldname": "error_log", + "fieldtype": "Long Text", + "label": "Error Log" } ], "index_web_pages_for_search": 1, "is_submittable": 1, "links": [], - "modified": "2024-02-05 10:36:34.229864", + "modified": "2024-03-19 17:04:49.369734", "modified_by": "Administrator", "module": "Setup", "name": "Transaction Deletion Record", diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py index 039d9a58413..b1e75117ad4 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py @@ -35,6 +35,7 @@ class TransactionDeletionRecord(Document): delete_transactions: DF.Check doctypes: DF.Table[TransactionDeletionRecordDetails] doctypes_to_be_ignored: DF.Table[TransactionDeletionRecordItem] + error_log: DF.LongText | None initialize_doctypes_table: DF.Check reset_company_default_values: DF.Check status: DF.Literal["Queued", "Running", "Failed", "Completed", "Cancelled"] @@ -146,11 +147,21 @@ class TransactionDeletionRecord(Document): task_to_execute=task, ) + # todo: add a non-background job based approach as well + def execute_task(self, task_to_execute: str | None = None): if task_to_execute: method = self.task_to_internal_method_map[task_to_execute] if task := getattr(self, method, None): - task() + try: + task() + except Exception as err: + frappe.db.rollback() + traceback = frappe.get_traceback(with_context=True) + if traceback: + message = "Traceback:
" + traceback + frappe.db.set_value(self.doctype, self.name, "error_log", message) + frappe.db.set_value(self.doctype, self.name, "status", "Failed") def delete_notifications(self): self.validate_doc_status() From 0455d0c46c8a171c0bbbd6f1730f671263cfedc4 Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Tue, 19 Mar 2024 17:41:37 +0530 Subject: [PATCH 22/28] refactor: minor UI tweaks --- .../transaction_deletion_record.js | 16 +++++++--------- .../transaction_deletion_record.py | 1 + 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.js b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.js index ccf09a6c38b..d543baa3cf6 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.js +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.js @@ -10,26 +10,24 @@ frappe.ui.form.on("Transaction Deletion Record", { callback: function (r) { doctypes_to_be_ignored_array = r.message; populate_doctypes_to_be_ignored(doctypes_to_be_ignored_array, frm); - frm.refresh_field('doctypes_to_be_ignored'); - } + frm.refresh_field("doctypes_to_be_ignored"); + }, }); } - }, - refresh: function(frm) { - if (frm.doc.docstatus==1 && ['Queued', 'Failed'].find(x => x == frm.doc.status)) { - let execute_btn = __("Start Deletion") + refresh: function (frm) { + if (frm.doc.docstatus == 1 && ["Queued", "Failed"].find((x) => x == frm.doc.status)) { + let execute_btn = frm.doc.status == "Queued" ? __("Start Deletion") : __("Retry"); frm.add_custom_button(execute_btn, () => { // Entry point for chain of events frm.call({ - method: 'process_tasks', - doc: frm.doc + method: "process_tasks", + doc: frm.doc, }); }); } - }, }); diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py index b1e75117ad4..1959a7d86d9 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py @@ -329,6 +329,7 @@ class TransactionDeletionRecord(Document): else: self.db_set("status", "Completed") self.db_set("delete_transactions", 1) + self.db_set("error_log", None) def get_doctypes_to_be_ignored_list(self): singles = frappe.get_all("DocType", filters={"issingle": 1}, pluck="name") From 3cec62d4f872a66557e1b9e56a2207f194e5439f Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Tue, 19 Mar 2024 17:49:05 +0530 Subject: [PATCH 23/28] chore: move status and error log to their own section --- .../transaction_deletion_record.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json index 8291a4ffc24..688b14a808a 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json @@ -7,10 +7,10 @@ "engine": "InnoDB", "field_order": [ "company", - "column_break_txbg", + "section_break_qpwb", "status", - "tasks_section", "error_log", + "tasks_section", "delete_bin_data", "delete_leads_and_addresses", "reset_company_default_values", @@ -63,10 +63,6 @@ "options": "Queued\nRunning\nFailed\nCompleted\nCancelled", "read_only": 1 }, - { - "fieldname": "column_break_txbg", - "fieldtype": "Column Break" - }, { "fieldname": "section_break_tbej", "fieldtype": "Section Break" @@ -129,12 +125,16 @@ "fieldname": "error_log", "fieldtype": "Long Text", "label": "Error Log" + }, + { + "fieldname": "section_break_qpwb", + "fieldtype": "Section Break" } ], "index_web_pages_for_search": 1, "is_submittable": 1, "links": [], - "modified": "2024-03-19 17:04:49.369734", + "modified": "2024-03-19 17:47:04.490196", "modified_by": "Administrator", "module": "Setup", "name": "Transaction Deletion Record", From 5fe0b20be108ce55a9519af5c953f7aebeaca93b Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Wed, 20 Mar 2024 14:10:08 +0530 Subject: [PATCH 24/28] chore: rename entry point --- .../transaction_deletion_record/transaction_deletion_record.js | 2 +- .../transaction_deletion_record/transaction_deletion_record.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.js b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.js index d543baa3cf6..9aa02784165 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.js +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.js @@ -23,7 +23,7 @@ frappe.ui.form.on("Transaction Deletion Record", { frm.add_custom_button(execute_btn, () => { // Entry point for chain of events frm.call({ - method: "process_tasks", + method: "start_deletion_tasks", doc: frm.doc, }); }); diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py index 1959a7d86d9..e8ffe55a2b7 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py @@ -198,7 +198,7 @@ class TransactionDeletionRecord(Document): ) @frappe.whitelist() - def process_tasks(self): + def start_deletion_tasks(self): # This method is the entry point for the chain of events that follow self.db_set("status", "Running") self.enqueue_task(task="Delete Bins") From 5a3afea8c772b7167839593ba88f3582a381259c Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Wed, 20 Mar 2024 14:32:27 +0530 Subject: [PATCH 25/28] refactor: link running doc validation to company master --- erpnext/setup/doctype/company/company.js | 2 +- erpnext/setup/doctype/company/company.py | 41 ++++++------------- .../transaction_deletion_record.py | 32 ++++++++++----- 3 files changed, 34 insertions(+), 41 deletions(-) diff --git a/erpnext/setup/doctype/company/company.js b/erpnext/setup/doctype/company/company.js index 39170053838..9b1a41a0b77 100644 --- a/erpnext/setup/doctype/company/company.js +++ b/erpnext/setup/doctype/company/company.js @@ -168,7 +168,7 @@ frappe.ui.form.on("Company", { delete_company_transactions: function (frm) { frappe.call({ - method: "erpnext.setup.doctype.company.company.is_deletion_job_running", + method: "erpnext.setup.doctype.transaction_deletion_record.transaction_deletion_record.is_deletion_doc_running", args: { company: frm.doc.name, }, diff --git a/erpnext/setup/doctype/company/company.py b/erpnext/setup/doctype/company/company.py index 876b6a4ac80..3ca14e65fdd 100644 --- a/erpnext/setup/doctype/company/company.py +++ b/erpnext/setup/doctype/company/company.py @@ -12,7 +12,6 @@ from frappe.contacts.address_and_contact import load_address_and_contact from frappe.custom.doctype.property_setter.property_setter import make_property_setter from frappe.desk.page.setup_wizard.setup_wizard import make_records from frappe.utils import cint, formatdate, get_link_to_form, get_timestamp, today -from frappe.utils.background_jobs import get_job, is_job_enqueued from frappe.utils.nestedset import NestedSet, rebuild_tree from erpnext.accounts.doctype.account.account import get_account_currency @@ -901,37 +900,21 @@ def get_default_company_address(name, sort_key="is_primary_address", existing_ad return None -def generate_id_for_deletion_job(company): - return "delete_company_transactions_" + company - - -@frappe.whitelist() -def is_deletion_job_running(company): - job_id = generate_id_for_deletion_job(company) - if is_job_enqueued(job_id): - job_name = get_job(job_id).get_id() # job name will have site prefix - frappe.throw( - _("A Transaction Deletion Job: {0} is already running for {1}").format( - frappe.bold(get_link_to_form("RQ Job", job_name)), frappe.bold(company) - ) - ) - - @frappe.whitelist() def create_transaction_deletion_request(company): - is_deletion_job_running(company) - job_id = generate_id_for_deletion_job(company) + from erpnext.setup.doctype.transaction_deletion_record.transaction_deletion_record import ( + is_deletion_doc_running, + ) + + is_deletion_doc_running(company) tdr = frappe.get_doc({"doctype": "Transaction Deletion Record", "company": company}) - tdr.insert() + tdr.submit() + tdr.start_deletion_tasks() - frappe.enqueue( - "frappe.utils.background_jobs.run_doc_method", - doctype=tdr.doctype, - name=tdr.name, - doc_method="submit", - job_id=job_id, - queue="long", - enqueue_after_commit=True, + frappe.msgprint( + _("A Transaction Deletion Document: {0} is triggered for {0}").format( + get_link_to_form("Transaction Deletion Record", tdr.name) + ), + frappe.bold(company), ) - frappe.msgprint(_("A Transaction Deletion Job is triggered for {0}").format(frappe.bold(company))) diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py index e8ffe55a2b7..4ee91306ee9 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py @@ -474,7 +474,25 @@ def get_doctypes_to_be_ignored(): return doctypes_to_be_ignored +@frappe.whitelist() +def is_deletion_doc_running(company: str | None = None, err_msg: str | None = None): + if company: + if running_deletion_jobs := frappe.db.get_all( + "Transaction Deletion Record", + filters={"docstatus": 1, "company": company, "status": "Running"}, + ): + if not err_msg: + err_msg = "" + frappe.throw( + title=_("Deletion in Progress!"), + msg=_("Transaction Deletion Document: {0} is running for this Company. {1}").format( + get_link_to_form("Transaction Deletion Record", running_deletion_jobs[0].name), err_msg + ), + ) + + def check_for_running_deletion_job(doc, method=None): + # Check if DocType has 'company' field df = qb.DocType("DocField") if ( not_allowed := qb.from_(df) @@ -482,14 +500,6 @@ def check_for_running_deletion_job(doc, method=None): .where((df.fieldname == "company") & (df.parent == doc.doctype)) .run() ): - if running_deletion_jobs := frappe.db.get_all( - "Transaction Deletion Record", - filters={"docstatus": 1, "company": doc.company, "status": "Running"}, - ): - frappe.throw( - _( - "Transaction Deletion job {0} is running for this Company. Cannot make any transactions until the deletion job is completed" - ).format( - get_link_to_form("Transaction Deletion Record", running_deletion_jobs[0].name) - ) - ) + is_deletion_doc_running( + doc.company, _("Cannot make any transactions until the deletion job is completed") + ) From a158b825d7eb359a66743b3e6972aa1b81389df0 Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Wed, 20 Mar 2024 14:59:19 +0530 Subject: [PATCH 26/28] refactor: ability to process in single transaction --- .../transaction_deletion_record.json | 11 ++++++-- .../transaction_deletion_record.py | 26 ++++++++++--------- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json index 688b14a808a..e03e1695e0e 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.json @@ -20,7 +20,8 @@ "section_break_tbej", "doctypes", "doctypes_to_be_ignored", - "amended_from" + "amended_from", + "process_in_single_transaction" ], "fields": [ { @@ -129,12 +130,18 @@ { "fieldname": "section_break_qpwb", "fieldtype": "Section Break" + }, + { + "default": "0", + "fieldname": "process_in_single_transaction", + "fieldtype": "Check", + "label": "Process in Single Transaction" } ], "index_web_pages_for_search": 1, "is_submittable": 1, "links": [], - "modified": "2024-03-19 17:47:04.490196", + "modified": "2024-03-20 14:58:15.086360", "modified_by": "Administrator", "module": "Setup", "name": "Transaction Deletion Record", diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py index 4ee91306ee9..00fad5f0fa6 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record.py @@ -37,6 +37,7 @@ class TransactionDeletionRecord(Document): doctypes_to_be_ignored: DF.Table[TransactionDeletionRecordItem] error_log: DF.LongText | None initialize_doctypes_table: DF.Check + process_in_single_transaction: DF.Check reset_company_default_values: DF.Check status: DF.Literal["Queued", "Running", "Failed", "Completed", "Cancelled"] # end: auto-generated types @@ -136,18 +137,19 @@ class TransactionDeletionRecord(Document): # Generate Job Id to uniquely identify each task for this document job_id = self.generate_job_name_for_task(task) - frappe.enqueue( - "frappe.utils.background_jobs.run_doc_method", - doctype=self.doctype, - name=self.name, - doc_method="execute_task", - job_id=job_id, - queue="long", - enqueue_after_commit=True, - task_to_execute=task, - ) - - # todo: add a non-background job based approach as well + if self.process_in_single_transaction: + self.execute_task(task_to_execute=task) + else: + frappe.enqueue( + "frappe.utils.background_jobs.run_doc_method", + doctype=self.doctype, + name=self.name, + doc_method="execute_task", + job_id=job_id, + queue="long", + enqueue_after_commit=True, + task_to_execute=task, + ) def execute_task(self, task_to_execute: str | None = None): if task_to_execute: From 81309576b0cb41ac2c91cf1abbf79b4655c7697d Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Wed, 20 Mar 2024 15:16:33 +0530 Subject: [PATCH 27/28] refactor(test): test cases modified to handle new approach --- erpnext/setup/demo.py | 2 ++ .../test_transaction_deletion_record.py | 3 +++ 2 files changed, 5 insertions(+) diff --git a/erpnext/setup/demo.py b/erpnext/setup/demo.py index 688d45a5a77..f48402e175b 100644 --- a/erpnext/setup/demo.py +++ b/erpnext/setup/demo.py @@ -181,8 +181,10 @@ def get_random_date(start_date, start_range, end_range): def create_transaction_deletion_record(company): transaction_deletion_record = frappe.new_doc("Transaction Deletion Record") transaction_deletion_record.company = company + transaction_deletion_record.process_in_single_transaction = True transaction_deletion_record.save(ignore_permissions=True) transaction_deletion_record.submit() + transaction_deletion_record.start_deletion_tasks() def clear_masters(): diff --git a/erpnext/setup/doctype/transaction_deletion_record/test_transaction_deletion_record.py b/erpnext/setup/doctype/transaction_deletion_record/test_transaction_deletion_record.py index 844e7865e3e..432438bcfee 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/test_transaction_deletion_record.py +++ b/erpnext/setup/doctype/transaction_deletion_record/test_transaction_deletion_record.py @@ -29,6 +29,7 @@ class TestTransactionDeletionRecord(FrappeTestCase): for i in range(5): create_task("Dunder Mifflin Paper Co") tdr = create_transaction_deletion_doc("Dunder Mifflin Paper Co") + tdr.reload() for doctype in tdr.doctypes: if doctype.doctype_name == "Task": self.assertEqual(doctype.no_of_docs, 5) @@ -60,7 +61,9 @@ def create_company(company_name): def create_transaction_deletion_doc(company): tdr = frappe.get_doc({"doctype": "Transaction Deletion Record", "company": company}) tdr.insert() + tdr.process_in_single_transaction = True tdr.submit() + tdr.start_deletion_tasks() return tdr From 02c522b7cddf0b1ae4bcc1d05e156a5b7aa09f2f Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Wed, 20 Mar 2024 16:01:09 +0530 Subject: [PATCH 28/28] chore: fix linting issue in JS --- .../transaction_deletion_record_list.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record_list.js b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record_list.js index 7c7b8ff25a7..285cb6dd221 100644 --- a/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record_list.js +++ b/erpnext/setup/doctype/transaction_deletion_record/transaction_deletion_record_list.js @@ -1,16 +1,16 @@ // Copyright (c) 2018, Frappe Technologies Pvt. Ltd. and Contributors // License: GNU General Public License v3. See license.txt -frappe.listview_settings['Transaction Deletion Record'] = { +frappe.listview_settings["Transaction Deletion Record"] = { add_fields: ["status"], - get_indicator: function(doc) { + get_indicator: function (doc) { let colors = { - 'Queued': 'orange', - 'Completed': 'green', - 'Running': 'blue', - 'Failed': 'red', + Queued: "orange", + Completed: "green", + Running: "blue", + Failed: "red", }; let status = doc.status; - return [__(status), colors[status], 'status,=,'+status]; + return [__(status), colors[status], "status,=," + status]; }, };