From 9f77793f1673ff9f55b8d181dc3dff7a68136da9 Mon Sep 17 00:00:00 2001 From: Diptanil Saha Date: Tue, 7 Jan 2025 17:28:46 +0530 Subject: [PATCH] chore: removal of tally migration feature (#45100) --- .../doctype/tally_migration/__init__.py | 0 .../tally_migration/tally_migration.js | 364 --------- .../tally_migration/tally_migration.json | 280 ------- .../tally_migration/tally_migration.py | 768 ------------------ .../tally_migration/test_tally_migration.py | 9 - 5 files changed, 1421 deletions(-) delete mode 100644 erpnext/erpnext_integrations/doctype/tally_migration/__init__.py delete mode 100644 erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js delete mode 100644 erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json delete mode 100644 erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py delete mode 100644 erpnext/erpnext_integrations/doctype/tally_migration/test_tally_migration.py diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/__init__.py b/erpnext/erpnext_integrations/doctype/tally_migration/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js deleted file mode 100644 index 556c332634d..00000000000 --- a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js +++ /dev/null @@ -1,364 +0,0 @@ -// Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and contributors -// For license information, please see license.txt - -frappe.provide("erpnext.tally_migration"); - -frappe.ui.form.on("Tally Migration", { - onload: function (frm) { - let reload_status = true; - frappe.realtime.on("tally_migration_progress_update", function (data) { - if (reload_status) { - frappe.model.with_doc(frm.doc.doctype, frm.doc.name, () => { - frm.refresh_header(); - }); - reload_status = false; - } - frm.dashboard.show_progress(data.title, (data.count / data.total) * 100, data.message); - let error_occurred = data.count === -1; - if (data.count == data.total || error_occurred) { - window.setTimeout( - (title) => { - frm.dashboard.hide_progress(title); - frm.reload_doc(); - if (error_occurred) { - frappe.msgprint({ - message: __("An error has occurred during {0}. Check {1} for more details", [ - repl( - "%(tally_document)s", - { - tally_document: frm.docname, - } - ), - "Error Log", - ]), - title: __("Tally Migration Error"), - indicator: "red", - }); - } - }, - 2000, - data.title - ); - } - }); - }, - - refresh: function (frm) { - frm.trigger("show_logs_preview"); - erpnext.tally_migration.failed_import_log = JSON.parse(frm.doc.failed_import_log); - erpnext.tally_migration.fixed_errors_log = JSON.parse(frm.doc.fixed_errors_log); - - ["default_round_off_account", "default_warehouse", "default_cost_center"].forEach((account) => { - frm.toggle_reqd(account, frm.doc.is_master_data_imported === 1); - frm.toggle_enable(account, frm.doc.is_day_book_data_processed != 1); - }); - - if (frm.doc.master_data && !frm.doc.is_master_data_imported) { - if (frm.doc.is_master_data_processed) { - if (frm.doc.status != "Importing Master Data") { - frm.events.add_button(frm, __("Import Master Data"), "import_master_data"); - } - } else { - if (frm.doc.status != "Processing Master Data") { - frm.events.add_button(frm, __("Process Master Data"), "process_master_data"); - } - } - } - - if (frm.doc.day_book_data && !frm.doc.is_day_book_data_imported) { - if (frm.doc.is_day_book_data_processed) { - if (frm.doc.status != "Importing Day Book Data") { - frm.events.add_button(frm, __("Import Day Book Data"), "import_day_book_data"); - } - } else { - if (frm.doc.status != "Processing Day Book Data") { - frm.events.add_button(frm, __("Process Day Book Data"), "process_day_book_data"); - } - } - } - }, - - erpnext_company: function (frm) { - frappe.db.exists("Company", frm.doc.erpnext_company).then((exists) => { - if (exists) { - frappe.msgprint( - __( - "Company {0} already exists. Continuing will overwrite the Company and Chart of Accounts", - [frm.doc.erpnext_company] - ) - ); - } - }); - }, - - add_button: function (frm, label, method) { - frm.add_custom_button(label, () => { - frm.call({ - doc: frm.doc, - method: method, - freeze: true, - }); - frm.reload_doc(); - }); - }, - - render_html_table(frm, shown_logs, hidden_logs, field) { - if (shown_logs && shown_logs.length > 0) { - frm.toggle_display(field, true); - } else { - frm.toggle_display(field, false); - return; - } - let rows = erpnext.tally_migration.get_html_rows(shown_logs, field); - let rows_head, table_caption; - - let table_footer = - hidden_logs && hidden_logs.length > 0 - ? ` - And ${hidden_logs.length} more others - ` - : ""; - - if (field === "fixed_error_log_preview") { - rows_head = `${__("Meta Data")} - ${__("Unresolve")}`; - table_caption = "Resolved Issues"; - } else { - rows_head = `${__("Error Message")} - ${__("Create")}`; - table_caption = "Error Log"; - } - - frm.get_field(field).$wrapper.html(` - - - - - - ${rows_head} - - ${rows} - ${table_footer} -
${table_caption}
${__("#")}${__("DocType")}
- `); - }, - - show_error_summary(frm) { - let summary = erpnext.tally_migration.failed_import_log.reduce((summary, row) => { - if (row.doc) { - if (summary[row.doc.doctype]) { - summary[row.doc.doctype] += 1; - } else { - summary[row.doc.doctype] = 1; - } - } - return summary; - }, {}); - console.table(summary); - }, - - show_logs_preview(frm) { - let empty = "[]"; - let import_log = frm.doc.failed_import_log || empty; - let completed_log = frm.doc.fixed_errors_log || empty; - let render_section = !(import_log === completed_log && import_log === empty); - - frm.toggle_display("import_log_section", render_section); - if (render_section) { - frm.trigger("show_error_summary"); - frm.trigger("show_errored_import_log"); - frm.trigger("show_fixed_errors_log"); - } - }, - - show_errored_import_log(frm) { - let import_log = erpnext.tally_migration.failed_import_log; - let logs = import_log.slice(0, 20); - let hidden_logs = import_log.slice(20); - - frm.events.render_html_table(frm, logs, hidden_logs, "failed_import_preview"); - }, - - show_fixed_errors_log(frm) { - let completed_log = erpnext.tally_migration.fixed_errors_log; - let logs = completed_log.slice(0, 20); - let hidden_logs = completed_log.slice(20); - - frm.events.render_html_table(frm, logs, hidden_logs, "fixed_error_log_preview"); - }, -}); - -erpnext.tally_migration.getError = (traceback) => { - /* Extracts the Error Message from the Python Traceback or Solved error */ - let is_multiline = traceback.trim().indexOf("\n") != -1; - let message; - - if (is_multiline) { - let exc_error_idx = traceback.trim().lastIndexOf("\n") + 1; - let error_line = traceback.substr(exc_error_idx); - let split_str_idx = error_line.indexOf(":") > 0 ? error_line.indexOf(":") + 1 : 0; - message = error_line.slice(split_str_idx).trim(); - } else { - message = traceback; - } - - return message; -}; - -erpnext.tally_migration.cleanDoc = (obj) => { - /* Strips all null and empty values of your JSON object */ - let temp = obj; - $.each(temp, function (key, value) { - if (value === "" || value === null) { - delete obj[key]; - } else if (Object.prototype.toString.call(value) === "[object Object]") { - erpnext.tally_migration.cleanDoc(value); - } else if ($.isArray(value)) { - $.each(value, function (k, v) { - erpnext.tally_migration.cleanDoc(v); - }); - } - }); - return temp; -}; - -erpnext.tally_migration.unresolve = (document) => { - /* Mark document migration as unresolved ie. move to failed error log */ - let frm = cur_frm; - let failed_log = erpnext.tally_migration.failed_import_log; - let fixed_log = erpnext.tally_migration.fixed_errors_log; - - let modified_fixed_log = fixed_log.filter((row) => { - if (!frappe.utils.deep_equal(erpnext.tally_migration.cleanDoc(row.doc), document)) { - return row; - } - }); - - failed_log.push({ doc: document, exc: `Marked unresolved on ${Date()}` }); - - frm.doc.failed_import_log = JSON.stringify(failed_log); - frm.doc.fixed_errors_log = JSON.stringify(modified_fixed_log); - - frm.dirty(); - frm.save(); -}; - -erpnext.tally_migration.resolve = (document) => { - /* Mark document migration as resolved ie. move to fixed error log */ - let frm = cur_frm; - let failed_log = erpnext.tally_migration.failed_import_log; - let fixed_log = erpnext.tally_migration.fixed_errors_log; - - let modified_failed_log = failed_log.filter((row) => { - if (!frappe.utils.deep_equal(erpnext.tally_migration.cleanDoc(row.doc), document)) { - return row; - } - }); - fixed_log.push({ doc: document, exc: `Solved on ${Date()}` }); - - frm.doc.failed_import_log = JSON.stringify(modified_failed_log); - frm.doc.fixed_errors_log = JSON.stringify(fixed_log); - - frm.dirty(); - frm.save(); -}; - -erpnext.tally_migration.create_new_doc = (document) => { - /* Mark as resolved and create new document */ - erpnext.tally_migration.resolve(document); - return frappe.call({ - type: "POST", - method: "erpnext.erpnext_integrations.doctype.tally_migration.tally_migration.new_doc", - args: { - document, - }, - freeze: true, - callback: function (r) { - if (!r.exc) { - frappe.model.sync(r.message); - frappe.get_doc(r.message.doctype, r.message.name).__run_link_triggers = true; - frappe.set_route("Form", r.message.doctype, r.message.name); - } - }, - }); -}; - -erpnext.tally_migration.get_html_rows = (logs, field) => { - let index = 0; - let rows = logs - .map(({ doc, exc }) => { - let id = frappe.dom.get_unique_id(); - let traceback = exc; - - let error_message = erpnext.tally_migration.getError(traceback); - index++; - - let show_traceback = ` - -
-
- 
${traceback}
-
-
`; - - let show_doc = ` - -
-
- 
${JSON.stringify(erpnext.tally_migration.cleanDoc(doc), null, 1)}
-
-
`; - - let create_button = ` - `; - - let mark_as_unresolved = ` - `; - - if (field === "fixed_error_log_preview") { - return ` - ${index} - -
${doc.doctype}
- - -
${error_message}
-
${show_doc}
- - -
${mark_as_unresolved}
- - `; - } else { - return ` - ${index} - -
${doc.doctype}
- - -
${error_message}
-
${show_traceback}
-
${show_doc}
- - -
${create_button}
- - `; - } - }) - .join(""); - - return rows; -}; diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json deleted file mode 100644 index e6df549ab41..00000000000 --- a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json +++ /dev/null @@ -1,280 +0,0 @@ -{ - "actions": [], - "beta": 1, - "creation": "2019-02-01 14:27:09.485238", - "doctype": "DocType", - "editable_grid": 1, - "engine": "InnoDB", - "field_order": [ - "status", - "master_data", - "is_master_data_processed", - "is_master_data_imported", - "column_break_2", - "tally_creditors_account", - "tally_debtors_account", - "company_section", - "tally_company", - "default_uom", - "column_break_8", - "erpnext_company", - "processed_files_section", - "chart_of_accounts", - "parties", - "addresses", - "column_break_17", - "uoms", - "items", - "vouchers", - "accounts_section", - "default_warehouse", - "default_round_off_account", - "column_break_21", - "default_cost_center", - "day_book_section", - "day_book_data", - "column_break_27", - "is_day_book_data_processed", - "is_day_book_data_imported", - "import_log_section", - "failed_import_log", - "fixed_errors_log", - "failed_import_preview", - "fixed_error_log_preview" - ], - "fields": [ - { - "fieldname": "status", - "fieldtype": "Data", - "hidden": 1, - "label": "Status" - }, - { - "description": "Data exported from Tally that consists of the Chart of Accounts, Customers, Suppliers, Addresses, Items and UOMs", - "fieldname": "master_data", - "fieldtype": "Attach", - "in_list_view": 1, - "label": "Master Data" - }, - { - "default": "Sundry Creditors", - "description": "Creditors Account set in Tally", - "fieldname": "tally_creditors_account", - "fieldtype": "Data", - "label": "Tally Creditors Account", - "read_only_depends_on": "eval:doc.is_master_data_processed==1", - "reqd": 1 - }, - { - "fieldname": "column_break_2", - "fieldtype": "Column Break" - }, - { - "default": "Sundry Debtors", - "description": "Debtors Account set in Tally", - "fieldname": "tally_debtors_account", - "fieldtype": "Data", - "label": "Tally Debtors Account", - "read_only_depends_on": "eval:doc.is_master_data_processed==1", - "reqd": 1 - }, - { - "depends_on": "is_master_data_processed", - "fieldname": "company_section", - "fieldtype": "Section Break" - }, - { - "description": "Company Name as per Imported Tally Data", - "fieldname": "tally_company", - "fieldtype": "Data", - "label": "Tally Company", - "read_only": 1 - }, - { - "fieldname": "column_break_8", - "fieldtype": "Column Break" - }, - { - "description": "Your Company set in ERPNext", - "fieldname": "erpnext_company", - "fieldtype": "Data", - "label": "ERPNext Company", - "read_only_depends_on": "eval:doc.is_master_data_processed==1" - }, - { - "fieldname": "processed_files_section", - "fieldtype": "Section Break", - "hidden": 1, - "label": "Processed Files" - }, - { - "fieldname": "chart_of_accounts", - "fieldtype": "Attach", - "label": "Chart of Accounts" - }, - { - "fieldname": "parties", - "fieldtype": "Attach", - "label": "Parties" - }, - { - "fieldname": "addresses", - "fieldtype": "Attach", - "label": "Addresses" - }, - { - "fieldname": "column_break_17", - "fieldtype": "Column Break" - }, - { - "fieldname": "uoms", - "fieldtype": "Attach", - "label": "UOMs" - }, - { - "fieldname": "items", - "fieldtype": "Attach", - "label": "Items" - }, - { - "fieldname": "vouchers", - "fieldtype": "Attach", - "label": "Vouchers" - }, - { - "depends_on": "is_master_data_imported", - "description": "The accounts are set by the system automatically but do confirm these defaults", - "fieldname": "accounts_section", - "fieldtype": "Section Break", - "label": "Accounts" - }, - { - "fieldname": "default_warehouse", - "fieldtype": "Link", - "label": "Default Warehouse", - "options": "Warehouse" - }, - { - "fieldname": "column_break_21", - "fieldtype": "Column Break" - }, - { - "fieldname": "default_cost_center", - "fieldtype": "Link", - "label": "Default Cost Center", - "options": "Cost Center" - }, - { - "default": "0", - "fieldname": "is_master_data_processed", - "fieldtype": "Check", - "label": "Is Master Data Processed", - "read_only": 1 - }, - { - "default": "0", - "fieldname": "is_day_book_data_processed", - "fieldtype": "Check", - "label": "Is Day Book Data Processed", - "read_only": 1 - }, - { - "default": "0", - "fieldname": "is_day_book_data_imported", - "fieldtype": "Check", - "label": "Is Day Book Data Imported", - "read_only": 1 - }, - { - "default": "0", - "fieldname": "is_master_data_imported", - "fieldtype": "Check", - "label": "Is Master Data Imported", - "read_only": 1 - }, - { - "depends_on": "is_master_data_imported", - "fieldname": "day_book_section", - "fieldtype": "Section Break" - }, - { - "fieldname": "column_break_27", - "fieldtype": "Column Break" - }, - { - "description": "Day Book Data exported from Tally that consists of all historic transactions", - "fieldname": "day_book_data", - "fieldtype": "Attach", - "in_list_view": 1, - "label": "Day Book Data" - }, - { - "default": "Unit", - "description": "UOM in case unspecified in imported data", - "fieldname": "default_uom", - "fieldtype": "Link", - "label": "Default UOM", - "options": "UOM", - "read_only_depends_on": "eval:doc.is_master_data_imported==1" - }, - { - "default": "[]", - "fieldname": "failed_import_log", - "fieldtype": "Code", - "hidden": 1, - "options": "JSON" - }, - { - "fieldname": "failed_import_preview", - "fieldtype": "HTML", - "label": "Failed Import Log" - }, - { - "fieldname": "import_log_section", - "fieldtype": "Section Break", - "label": "Import Log" - }, - { - "fieldname": "default_round_off_account", - "fieldtype": "Link", - "label": "Default Round Off Account", - "options": "Account" - }, - { - "default": "[]", - "fieldname": "fixed_errors_log", - "fieldtype": "Code", - "hidden": 1, - "options": "JSON" - }, - { - "fieldname": "fixed_error_log_preview", - "fieldtype": "HTML", - "label": "Fixed Error Log" - } - ], - "links": [], - "modified": "2024-03-27 13:10:51.146772", - "modified_by": "Administrator", - "module": "ERPNext Integrations", - "name": "Tally Migration", - "owner": "Administrator", - "permissions": [ - { - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "share": 1, - "write": 1 - } - ], - "sort_field": "creation", - "sort_order": "DESC", - "states": [], - "track_changes": 1 -} \ No newline at end of file diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py deleted file mode 100644 index c811b3832e7..00000000000 --- a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py +++ /dev/null @@ -1,768 +0,0 @@ -# Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and contributors -# For license information, please see license.txt - - -import json -import re -import sys -import traceback -import zipfile -from decimal import Decimal - -import frappe -from bs4 import BeautifulSoup as bs -from frappe import _ -from frappe.custom.doctype.custom_field.custom_field import ( - create_custom_fields as _create_custom_fields, -) -from frappe.model.document import Document -from frappe.utils.data import format_datetime - -from erpnext import encode_company_abbr -from erpnext.accounts.doctype.account.chart_of_accounts.chart_of_accounts import create_charts -from erpnext.accounts.doctype.chart_of_accounts_importer.chart_of_accounts_importer import ( - unset_existing_data, -) - -PRIMARY_ACCOUNT = "Primary" -VOUCHER_CHUNK_SIZE = 500 - - -@frappe.whitelist() -def new_doc(document): - document = json.loads(document) - doctype = document.pop("doctype") - document.pop("name", None) - doc = frappe.new_doc(doctype) - doc.update(document) - - return doc - - -class TallyMigration(Document): - # begin: auto-generated types - # This code is auto-generated. Do not modify anything in this block. - - from typing import TYPE_CHECKING - - if TYPE_CHECKING: - from frappe.types import DF - - addresses: DF.Attach | None - chart_of_accounts: DF.Attach | None - day_book_data: DF.Attach | None - default_cost_center: DF.Link | None - default_round_off_account: DF.Link | None - default_uom: DF.Link | None - default_warehouse: DF.Link | None - erpnext_company: DF.Data | None - failed_import_log: DF.Code | None - fixed_errors_log: DF.Code | None - is_day_book_data_imported: DF.Check - is_day_book_data_processed: DF.Check - is_master_data_imported: DF.Check - is_master_data_processed: DF.Check - items: DF.Attach | None - master_data: DF.Attach | None - parties: DF.Attach | None - status: DF.Data | None - tally_company: DF.Data | None - tally_creditors_account: DF.Data - tally_debtors_account: DF.Data - uoms: DF.Attach | None - vouchers: DF.Attach | None - # end: auto-generated types - - def validate(self): - failed_import_log = json.loads(self.failed_import_log) - sorted_failed_import_log = sorted(failed_import_log, key=lambda row: row["doc"]["creation"]) - self.failed_import_log = json.dumps(sorted_failed_import_log) - - def autoname(self): - if not self.name: - self.name = "Tally Migration on " + format_datetime(self.creation) - - def get_collection(self, data_file): - def sanitize(string): - return re.sub("", "", string) - - def emptify(string): - string = re.sub(r"<\w+/>", "", string) - string = re.sub(r"<([\w.]+)>\s*<\/\1>", "", string) - string = re.sub(r"\r\n", "", string) - return string - - master_file = frappe.get_doc("File", {"file_url": data_file}) - master_file_path = master_file.get_full_path() - - if zipfile.is_zipfile(master_file_path): - with zipfile.ZipFile(master_file_path) as zf: - encoded_content = zf.read(zf.namelist()[0]) - try: - content = encoded_content.decode("utf-8-sig") - except UnicodeDecodeError: - content = encoded_content.decode("utf-16") - - master = bs(sanitize(emptify(content)), "xml") - collection = master.BODY.IMPORTDATA.REQUESTDATA - return collection - - def dump_processed_data(self, data): - for key, value in data.items(): - f = frappe.get_doc( - { - "doctype": "File", - "file_name": key + ".json", - "attached_to_doctype": self.doctype, - "attached_to_name": self.name, - "content": json.dumps(value), - "is_private": True, - } - ) - try: - f.insert(ignore_if_duplicate=True) - except frappe.DuplicateEntryError: - pass - setattr(self, key, f.file_url) - - def set_account_defaults(self): - self.default_cost_center, self.default_round_off_account = frappe.db.get_value( - "Company", self.erpnext_company, ["cost_center", "round_off_account"] - ) - self.default_warehouse = frappe.db.get_single_value("Stock Settings", "default_warehouse") - - def _process_master_data(self): - def get_company_name(collection): - return collection.find_all("REMOTECMPINFO.LIST")[0].REMOTECMPNAME.string.strip() - - def get_coa_customers_suppliers(collection): - root_type_map = { - "Application of Funds (Assets)": "Asset", - "Expenses": "Expense", - "Income": "Income", - "Source of Funds (Liabilities)": "Liability", - } - roots = set(root_type_map.keys()) - accounts = list(get_groups(collection.find_all("GROUP"))) + list( - get_ledgers(collection.find_all("LEDGER")) - ) - children, parents = get_children_and_parent_dict(accounts) - group_set = [acc[1] for acc in accounts if acc[2]] - children, customers, suppliers = remove_parties(parents, children, group_set) - - try: - coa = traverse({}, children, roots, roots, group_set) - except RecursionError: - self.log( - _( - "Error occurred while parsing Chart of Accounts: Please make sure that no two accounts have the same name" - ) - ) - - for account in coa: - coa[account]["root_type"] = root_type_map[account] - - return coa, customers, suppliers - - def get_groups(accounts): - for account in accounts: - if account["NAME"] in (self.tally_creditors_account, self.tally_debtors_account): - yield get_parent(account), account["NAME"], 0 - else: - yield get_parent(account), account["NAME"], 1 - - def get_ledgers(accounts): - for account in accounts: - # If Ledger doesn't have PARENT field then don't create Account - # For example "Profit & Loss A/c" - if account.PARENT: - yield account.PARENT.string.strip(), account["NAME"], 0 - - def get_parent(account): - if account.PARENT: - return account.PARENT.string.strip() - return { - ("Yes", "No"): "Application of Funds (Assets)", - ("Yes", "Yes"): "Expenses", - ("No", "Yes"): "Income", - ("No", "No"): "Source of Funds (Liabilities)", - }[(account.ISDEEMEDPOSITIVE.string.strip(), account.ISREVENUE.string.strip())] - - def get_children_and_parent_dict(accounts): - children, parents = {}, {} - for parent, account, _is_group in accounts: - children.setdefault(parent, set()).add(account) - parents.setdefault(account, set()).add(parent) - parents[account].update(parents.get(parent, [])) - return children, parents - - def remove_parties(parents, children, group_set): - customers, suppliers = set(), set() - for account in parents: - found = False - if self.tally_creditors_account in parents[account]: - found = True - if account not in group_set: - suppliers.add(account) - if self.tally_debtors_account in parents[account]: - found = True - if account not in group_set: - customers.add(account) - if found: - children.pop(account, None) - - return children, customers, suppliers - - def traverse(tree, children, accounts, roots, group_set): - for account in accounts: - if account in group_set or account in roots: - if account in children: - tree[account] = traverse({}, children, children[account], roots, group_set) - else: - tree[account] = {"is_group": 1} - else: - tree[account] = {} - return tree - - def get_parties_addresses(collection, customers, suppliers): - parties, addresses = [], [] - for account in collection.find_all("LEDGER"): - party_type = None - links = [] - if account.NAME.string.strip() in customers: - party_type = "Customer" - parties.append( - { - "doctype": party_type, - "customer_name": account.NAME.string.strip(), - "tax_id": account.INCOMETAXNUMBER.string.strip() - if account.INCOMETAXNUMBER - else None, - "customer_group": "All Customer Groups", - "territory": "All Territories", - "customer_type": "Individual", - } - ) - links.append({"link_doctype": party_type, "link_name": account["NAME"]}) - - if account.NAME.string.strip() in suppliers: - party_type = "Supplier" - parties.append( - { - "doctype": party_type, - "supplier_name": account.NAME.string.strip(), - "pan": account.INCOMETAXNUMBER.string.strip() - if account.INCOMETAXNUMBER - else None, - "supplier_group": "All Supplier Groups", - "supplier_type": "Individual", - } - ) - links.append({"link_doctype": party_type, "link_name": account["NAME"]}) - - if party_type: - address = "\n".join([a.string.strip() for a in account.find_all("ADDRESS")]) - addresses.append( - { - "doctype": "Address", - "address_line1": address[:140].strip(), - "address_line2": address[140:].strip(), - "country": account.COUNTRYNAME.string.strip() if account.COUNTRYNAME else None, - "state": account.LEDSTATENAME.string.strip() if account.LEDSTATENAME else None, - "gst_state": account.LEDSTATENAME.string.strip() - if account.LEDSTATENAME - else None, - "pin_code": account.PINCODE.string.strip() if account.PINCODE else None, - "mobile": account.LEDGERPHONE.string.strip() if account.LEDGERPHONE else None, - "phone": account.LEDGERPHONE.string.strip() if account.LEDGERPHONE else None, - "gstin": account.PARTYGSTIN.string.strip() if account.PARTYGSTIN else None, - "links": links, - } - ) - return parties, addresses - - def get_stock_items_uoms(collection): - uoms = [] - for uom in collection.find_all("UNIT"): - uoms.append({"doctype": "UOM", "uom_name": uom.NAME.string.strip()}) - - items = [] - for item in collection.find_all("STOCKITEM"): - stock_uom = item.BASEUNITS.string.strip() if item.BASEUNITS else self.default_uom - items.append( - { - "doctype": "Item", - "item_code": item.NAME.string.strip(), - "stock_uom": stock_uom.strip(), - "is_stock_item": 0, - "item_group": "All Item Groups", - "item_defaults": [{"company": self.erpnext_company}], - } - ) - - return items, uoms - - try: - self.publish("Process Master Data", _("Reading Uploaded File"), 1, 5) - collection = self.get_collection(self.master_data) - company = get_company_name(collection) - self.tally_company = company - self.erpnext_company = company - - self.publish("Process Master Data", _("Processing Chart of Accounts and Parties"), 2, 5) - chart_of_accounts, customers, suppliers = get_coa_customers_suppliers(collection) - - self.publish("Process Master Data", _("Processing Party Addresses"), 3, 5) - parties, addresses = get_parties_addresses(collection, customers, suppliers) - - self.publish("Process Master Data", _("Processing Items and UOMs"), 4, 5) - items, uoms = get_stock_items_uoms(collection) - data = { - "chart_of_accounts": chart_of_accounts, - "parties": parties, - "addresses": addresses, - "items": items, - "uoms": uoms, - } - - self.publish("Process Master Data", _("Done"), 5, 5) - self.dump_processed_data(data) - - self.is_master_data_processed = 1 - - except Exception: - self.publish("Process Master Data", _("Process Failed"), -1, 5) - self.log() - - finally: - self.set_status() - - def publish(self, title, message, count, total): - frappe.publish_realtime( - "tally_migration_progress_update", - {"title": title, "message": message, "count": count, "total": total}, - user=self.modified_by, - ) - - def _import_master_data(self): - def create_company_and_coa(coa_file_url): - coa_file = frappe.get_doc("File", {"file_url": coa_file_url}) - frappe.local.flags.ignore_chart_of_accounts = True - - try: - company = frappe.get_doc( - { - "doctype": "Company", - "company_name": self.erpnext_company, - "default_currency": "INR", - "enable_perpetual_inventory": 0, - } - ).insert() - except frappe.DuplicateEntryError: - company = frappe.get_doc("Company", self.erpnext_company) - unset_existing_data(self.erpnext_company) - - frappe.local.flags.ignore_chart_of_accounts = False - create_charts(company.name, custom_chart=json.loads(coa_file.get_content())) - company.create_default_warehouses() - - def create_parties_and_addresses(parties_file_url, addresses_file_url): - parties_file = frappe.get_doc("File", {"file_url": parties_file_url}) - for party in json.loads(parties_file.get_content()): - try: - party_doc = frappe.get_doc(party) - party_doc.insert() - except Exception: - self.log(party_doc) - addresses_file = frappe.get_doc("File", {"file_url": addresses_file_url}) - for address in json.loads(addresses_file.get_content()): - try: - address_doc = frappe.get_doc(address) - address_doc.insert(ignore_mandatory=True) - except Exception: - self.log(address_doc) - - def create_items_uoms(items_file_url, uoms_file_url): - uoms_file = frappe.get_doc("File", {"file_url": uoms_file_url}) - for uom in json.loads(uoms_file.get_content()): - if not frappe.db.exists(uom): - try: - uom_doc = frappe.get_doc(uom) - uom_doc.insert() - except Exception: - self.log(uom_doc) - - items_file = frappe.get_doc("File", {"file_url": items_file_url}) - for item in json.loads(items_file.get_content()): - try: - item_doc = frappe.get_doc(item) - item_doc.insert() - except Exception: - self.log(item_doc) - - try: - self.publish("Import Master Data", _("Creating Company and Importing Chart of Accounts"), 1, 4) - create_company_and_coa(self.chart_of_accounts) - - self.publish("Import Master Data", _("Importing Parties and Addresses"), 2, 4) - create_parties_and_addresses(self.parties, self.addresses) - - self.publish("Import Master Data", _("Importing Items and UOMs"), 3, 4) - create_items_uoms(self.items, self.uoms) - - self.publish("Import Master Data", _("Done"), 4, 4) - - self.set_account_defaults() - self.is_master_data_imported = 1 - frappe.db.commit() - - except Exception: - self.publish("Import Master Data", _("Process Failed"), -1, 5) - frappe.db.rollback() - self.log() - - finally: - self.set_status() - - def _process_day_book_data(self): - def get_vouchers(collection): - vouchers = [] - for voucher in collection.find_all("VOUCHER"): - if voucher.ISCANCELLED.string.strip() == "Yes": - continue - inventory_entries = ( - voucher.find_all("INVENTORYENTRIES.LIST") - + voucher.find_all("ALLINVENTORYENTRIES.LIST") - + voucher.find_all("INVENTORYENTRIESIN.LIST") - + voucher.find_all("INVENTORYENTRIESOUT.LIST") - ) - if ( - voucher.VOUCHERTYPENAME.string.strip() not in ["Journal", "Receipt", "Payment", "Contra"] - and inventory_entries - ): - function = voucher_to_invoice - else: - function = voucher_to_journal_entry - try: - processed_voucher = function(voucher) - if processed_voucher: - vouchers.append(processed_voucher) - frappe.db.commit() - except Exception: - frappe.db.rollback() - self.log(voucher) - return vouchers - - def voucher_to_journal_entry(voucher): - accounts = [] - ledger_entries = voucher.find_all("ALLLEDGERENTRIES.LIST") + voucher.find_all( - "LEDGERENTRIES.LIST" - ) - for entry in ledger_entries: - account = { - "account": encode_company_abbr(entry.LEDGERNAME.string.strip(), self.erpnext_company), - "cost_center": self.default_cost_center, - } - if entry.ISPARTYLEDGER.string.strip() == "Yes": - party_details = get_party(entry.LEDGERNAME.string.strip()) - if party_details: - party_type, party_account = party_details - account["party_type"] = party_type - account["account"] = party_account - account["party"] = entry.LEDGERNAME.string.strip() - amount = Decimal(entry.AMOUNT.string.strip()) - if amount > 0: - account["credit_in_account_currency"] = str(abs(amount)) - else: - account["debit_in_account_currency"] = str(abs(amount)) - accounts.append(account) - - journal_entry = { - "doctype": "Journal Entry", - "tally_guid": voucher.GUID.string.strip(), - "tally_voucher_no": voucher.VOUCHERNUMBER.string.strip() if voucher.VOUCHERNUMBER else "", - "posting_date": voucher.DATE.string.strip(), - "company": self.erpnext_company, - "accounts": accounts, - } - return journal_entry - - def voucher_to_invoice(voucher): - if voucher.VOUCHERTYPENAME.string.strip() in ["Sales", "Credit Note"]: - doctype = "Sales Invoice" - party_field = "customer" - account_field = "debit_to" - account_name = encode_company_abbr(self.tally_debtors_account, self.erpnext_company) - price_list_field = "selling_price_list" - elif voucher.VOUCHERTYPENAME.string.strip() in ["Purchase", "Debit Note"]: - doctype = "Purchase Invoice" - party_field = "supplier" - account_field = "credit_to" - account_name = encode_company_abbr(self.tally_creditors_account, self.erpnext_company) - price_list_field = "buying_price_list" - else: - # Do not handle vouchers other than "Purchase", "Debit Note", "Sales" and "Credit Note" - # Do not handle Custom Vouchers either - return - - invoice = { - "doctype": doctype, - party_field: voucher.PARTYNAME.string.strip(), - "tally_guid": voucher.GUID.string.strip(), - "tally_voucher_no": voucher.VOUCHERNUMBER.string.strip() if voucher.VOUCHERNUMBER else "", - "posting_date": voucher.DATE.string.strip(), - "due_date": voucher.DATE.string.strip(), - "items": get_voucher_items(voucher, doctype), - "taxes": get_voucher_taxes(voucher), - account_field: account_name, - price_list_field: "Tally Price List", - "set_posting_time": 1, - "disable_rounded_total": 1, - "company": self.erpnext_company, - } - return invoice - - def get_voucher_items(voucher, doctype): - inventory_entries = ( - voucher.find_all("INVENTORYENTRIES.LIST") - + voucher.find_all("ALLINVENTORYENTRIES.LIST") - + voucher.find_all("INVENTORYENTRIESIN.LIST") - + voucher.find_all("INVENTORYENTRIESOUT.LIST") - ) - if doctype == "Sales Invoice": - account_field = "income_account" - elif doctype == "Purchase Invoice": - account_field = "expense_account" - items = [] - for entry in inventory_entries: - qty, uom = entry.ACTUALQTY.string.strip().split() - items.append( - { - "item_code": entry.STOCKITEMNAME.string.strip(), - "description": entry.STOCKITEMNAME.string.strip(), - "qty": qty.strip(), - "uom": uom.strip(), - "conversion_factor": 1, - "price_list_rate": entry.RATE.string.strip().split("/")[0], - "cost_center": self.default_cost_center, - "warehouse": self.default_warehouse, - account_field: encode_company_abbr( - entry.find_all("ACCOUNTINGALLOCATIONS.LIST")[0].LEDGERNAME.string.strip(), - self.erpnext_company, - ), - } - ) - return items - - def get_voucher_taxes(voucher): - ledger_entries = voucher.find_all("ALLLEDGERENTRIES.LIST") + voucher.find_all( - "LEDGERENTRIES.LIST" - ) - taxes = [] - for entry in ledger_entries: - if entry.ISPARTYLEDGER.string.strip() == "No": - tax_account = encode_company_abbr(entry.LEDGERNAME.string.strip(), self.erpnext_company) - taxes.append( - { - "charge_type": "Actual", - "account_head": tax_account, - "description": tax_account, - "tax_amount": entry.AMOUNT.string.strip(), - "cost_center": self.default_cost_center, - } - ) - return taxes - - def get_party(party): - if frappe.db.exists({"doctype": "Supplier", "supplier_name": party}): - return "Supplier", encode_company_abbr(self.tally_creditors_account, self.erpnext_company) - elif frappe.db.exists({"doctype": "Customer", "customer_name": party}): - return "Customer", encode_company_abbr(self.tally_debtors_account, self.erpnext_company) - - try: - self.publish("Process Day Book Data", _("Reading Uploaded File"), 1, 3) - collection = self.get_collection(self.day_book_data) - - self.publish("Process Day Book Data", _("Processing Vouchers"), 2, 3) - vouchers = get_vouchers(collection) - - self.publish("Process Day Book Data", _("Done"), 3, 3) - self.dump_processed_data({"vouchers": vouchers}) - - self.is_day_book_data_processed = 1 - - except Exception: - self.publish("Process Day Book Data", _("Process Failed"), -1, 5) - self.log() - - finally: - self.set_status() - - def _import_day_book_data(self): - def create_fiscal_years(vouchers): - from frappe.utils.data import add_years, getdate - - earliest_date = getdate(min(voucher["posting_date"] for voucher in vouchers)) - oldest_year = frappe.get_all( - "Fiscal Year", fields=["year_start_date", "year_end_date"], order_by="year_start_date" - )[0] - while earliest_date < oldest_year.year_start_date: - new_year = frappe.get_doc({"doctype": "Fiscal Year"}) - new_year.year_start_date = add_years(oldest_year.year_start_date, -1) - new_year.year_end_date = add_years(oldest_year.year_end_date, -1) - if new_year.year_start_date.year == new_year.year_end_date.year: - new_year.year = new_year.year_start_date.year - else: - new_year.year = f"{new_year.year_start_date.year}-{new_year.year_end_date.year}" - new_year.save() - oldest_year = new_year - - def create_custom_fields(): - _create_custom_fields( - { - ("Journal Entry", "Purchase Invoice", "Sales Invoice"): [ - { - "fieldtype": "Data", - "fieldname": "tally_guid", - "read_only": 1, - "label": "Tally GUID", - }, - { - "fieldtype": "Data", - "fieldname": "tally_voucher_no", - "read_only": 1, - "label": "Tally Voucher Number", - }, - ] - } - ) - - def create_price_list(): - frappe.get_doc( - { - "doctype": "Price List", - "price_list_name": "Tally Price List", - "selling": 1, - "buying": 1, - "enabled": 1, - "currency": "INR", - } - ).insert() - - try: - frappe.db.set_value( - "Account", - encode_company_abbr(self.tally_creditors_account, self.erpnext_company), - "account_type", - "Payable", - ) - frappe.db.set_value( - "Account", - encode_company_abbr(self.tally_debtors_account, self.erpnext_company), - "account_type", - "Receivable", - ) - frappe.db.set_value( - "Company", self.erpnext_company, "round_off_account", self.default_round_off_account - ) - - vouchers_file = frappe.get_doc("File", {"file_url": self.vouchers}) - vouchers = json.loads(vouchers_file.get_content()) - - create_fiscal_years(vouchers) - create_price_list() - create_custom_fields() - - total = len(vouchers) - is_last = False - - for index in range(0, total, VOUCHER_CHUNK_SIZE): - if index + VOUCHER_CHUNK_SIZE >= total: - is_last = True - frappe.enqueue_doc( - self.doctype, - self.name, - "_import_vouchers", - queue="long", - timeout=3600, - start=index + 1, - total=total, - is_last=is_last, - ) - - except Exception: - self.log() - - finally: - self.set_status() - - def _import_vouchers(self, start, total, is_last=False): - frappe.flags.in_migrate = True - vouchers_file = frappe.get_doc("File", {"file_url": self.vouchers}) - vouchers = json.loads(vouchers_file.get_content()) - chunk = vouchers[start : start + VOUCHER_CHUNK_SIZE] - - for index, voucher in enumerate(chunk, start=start): - try: - voucher_doc = frappe.get_doc(voucher) - voucher_doc.insert() - voucher_doc.submit() - self.publish("Importing Vouchers", _("{} of {}").format(index, total), index, total) - frappe.db.commit() - except Exception: - frappe.db.rollback() - self.log(voucher_doc) - - if is_last: - self.status = "" - self.is_day_book_data_imported = 1 - self.save() - frappe.db.set_value("Price List", "Tally Price List", "enabled", 0) - frappe.flags.in_migrate = False - - @frappe.whitelist() - def process_master_data(self): - self.set_status("Processing Master Data") - frappe.enqueue_doc(self.doctype, self.name, "_process_master_data", queue="long", timeout=3600) - - @frappe.whitelist() - def import_master_data(self): - self.set_status("Importing Master Data") - frappe.enqueue_doc(self.doctype, self.name, "_import_master_data", queue="long", timeout=3600) - - @frappe.whitelist() - def process_day_book_data(self): - self.set_status("Processing Day Book Data") - frappe.enqueue_doc(self.doctype, self.name, "_process_day_book_data", queue="long", timeout=3600) - - @frappe.whitelist() - def import_day_book_data(self): - self.set_status("Importing Day Book Data") - frappe.enqueue_doc(self.doctype, self.name, "_import_day_book_data", queue="long", timeout=3600) - - def log(self, data=None): - if isinstance(data, frappe.model.document.Document): - if sys.exc_info()[1].__class__ != frappe.DuplicateEntryError: - failed_import_log = json.loads(self.failed_import_log) - doc = data.as_dict() - failed_import_log.append({"doc": doc, "exc": traceback.format_exc()}) - self.failed_import_log = json.dumps(failed_import_log, separators=(",", ":")) - self.save() - frappe.db.commit() - - else: - data = data or self.status - message = "\n".join( - [ - "Data:", - json.dumps(data, default=str, indent=4), - "--" * 50, - "\nException:", - traceback.format_exc(), - ] - ) - return frappe.log_error(title="Tally Migration Error", message=message) - - def set_status(self, status=""): - self.status = status - self.save() diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/test_tally_migration.py b/erpnext/erpnext_integrations/doctype/tally_migration/test_tally_migration.py deleted file mode 100644 index 62c52e51b31..00000000000 --- a/erpnext/erpnext_integrations/doctype/tally_migration/test_tally_migration.py +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and Contributors -# See license.txt -import unittest - -from frappe.tests import IntegrationTestCase - - -class TestTallyMigration(IntegrationTestCase): - pass