From 88c2ba54abc4357e4cff623447e8fad6d140b5a1 Mon Sep 17 00:00:00 2001 From: Deepesh Garg Date: Sun, 23 Feb 2020 21:03:47 +0530 Subject: [PATCH] fix: Excel support and UX fixes for chart of accounts importer --- .../chart_of_accounts_importer.js | 58 ++-- .../chart_of_accounts_importer.json | 267 ++++-------------- .../chart_of_accounts_importer.py | 166 +++++++++-- 3 files changed, 235 insertions(+), 256 deletions(-) diff --git a/erpnext/accounts/doctype/chart_of_accounts_importer/chart_of_accounts_importer.js b/erpnext/accounts/doctype/chart_of_accounts_importer/chart_of_accounts_importer.js index 40a97ae295e..93a4180a30a 100644 --- a/erpnext/accounts/doctype/chart_of_accounts_importer/chart_of_accounts_importer.js +++ b/erpnext/accounts/doctype/chart_of_accounts_importer/chart_of_accounts_importer.js @@ -2,6 +2,12 @@ frappe.ui.form.on('Chart of Accounts Importer', { onload: function (frm) { frm.set_value("company", ""); frm.set_value("import_file", ""); + + frappe.db.get_value('File', {'attached_to_doctype': 'Chart of Accounts Importer'}, 'file_url', (r) => { + if (r) { + frm.set_value('import_file', r.file_url); + } + }); }, refresh: function (frm) { // disable default save @@ -17,17 +23,15 @@ frappe.ui.form.on('Chart of Accounts Importer', { if (frm.page && frm.page.show_import_button) { create_import_button(frm); } + }, - // show download template button when company is properly selected - if(frm.doc.company) { - // download the csv template file - frm.add_custom_button(__("Download template"), function () { - let get_template_url = 'erpnext.accounts.doctype.chart_of_accounts_importer.chart_of_accounts_importer.download_template'; - open_url_post(frappe.request.url, { cmd: get_template_url, doctype: frm.doc.doctype }); - }); - } else { - frm.set_value("import_file", ""); - } + download_template: function(frm) { + open_url_post( + '/api/method/erpnext.accounts.doctype.chart_of_accounts_importer.chart_of_accounts_importer.download_template', + { + file_type: frm.doc.file_type + } + ) }, import_file: function (frm) { @@ -41,21 +45,23 @@ frappe.ui.form.on('Chart of Accounts Importer', { }, company: function (frm) { - // validate that no Gl Entry record for the company exists. - frappe.call({ - method: "erpnext.accounts.doctype.chart_of_accounts_importer.chart_of_accounts_importer.validate_company", - args: { - company: frm.doc.company - }, - callback: function(r) { - if(r.message===false) { - frm.set_value("company", ""); - frappe.throw(__("Transactions against the company already exist! ")); - } else { - frm.trigger("refresh"); + if (frm.doc.company) { + // validate that no Gl Entry record for the company exists. + frappe.call({ + method: "erpnext.accounts.doctype.chart_of_accounts_importer.chart_of_accounts_importer.validate_company", + args: { + company: frm.doc.company + }, + callback: function(r) { + if(r.message===false) { + frm.set_value("company", ""); + frappe.throw(__("Transactions against the company already exist! ")); + } else { + frm.trigger("refresh"); + } } - } - }); + }); + } } }); @@ -87,6 +93,7 @@ var create_import_button = function(frm) { freeze: true, freeze_message: __("Creating Accounts..."), callback: function(r) { + console.log(r, '###########'); if(!r.exc) { clearInterval(frm.page["interval"]); frm.page.set_indicator(__('Import Successfull'), 'blue'); @@ -118,7 +125,8 @@ var generate_tree_preview = function(frm) { args: { file_name: frm.doc.import_file, parent: parent, - doctype: 'Chart of Accounts Importer' + doctype: 'Chart of Accounts Importer', + file_type: frm.doc.file_type }, onclick: function(node) { parent = node.value; diff --git a/erpnext/accounts/doctype/chart_of_accounts_importer/chart_of_accounts_importer.json b/erpnext/accounts/doctype/chart_of_accounts_importer/chart_of_accounts_importer.json index d544e69231e..555413b31fa 100644 --- a/erpnext/accounts/doctype/chart_of_accounts_importer/chart_of_accounts_importer.json +++ b/erpnext/accounts/doctype/chart_of_accounts_importer/chart_of_accounts_importer.json @@ -1,226 +1,83 @@ { - "allow_copy": 1, - "allow_events_in_timeline": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2019-02-01 12:24:34.761380", - "custom": 0, + "actions": [], + "allow_copy": 1, + "creation": "2019-02-01 12:24:34.761380", "description": "Import Chart of Accounts from a csv file", - "docstatus": 0, - "doctype": "DocType", - "document_type": "Other", - "editable_grid": 1, - "engine": "InnoDB", + "doctype": "DocType", + "document_type": "Other", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "company", + "file_type", + "download_template", + "import_file_section", + "import_file", + "chart_preview", + "chart_tree" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "company", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Company", - "length": 0, - "no_copy": 0, - "options": "Company", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "company", + "fieldtype": "Link", + "in_list_view": 1, + "label": "Company", + "options": "Company" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "import_file_section", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldtype": "Section Break" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "", + "depends_on": "eval:!doc.__islocal", "fieldname": "import_file", "fieldtype": "Attach", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Attach custom Chart of Accounts file", - "length": 0, - "no_copy": 0, - "options": "", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "label": "Attach custom Chart of Accounts file" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, "collapsible": 1, - "columns": 0, "fieldname": "chart_preview", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Chart Preview", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldtype": "Section Break", + "label": "Chart Preview" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "chart_tree", "fieldtype": "HTML", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Chart Tree", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "label": "Chart Tree" + }, + { + "depends_on": "company", + "fieldname": "download_template", + "fieldtype": "Button", + "label": "Download Template" + }, + { + "depends_on": "company", + "fieldname": "file_type", + "fieldtype": "Select", + "label": "File Type", + "options": "CSV\nExcel" } - ], - "has_web_view": 0, - "hide_heading": 1, - "hide_toolbar": 1, - "idx": 0, - "image_view": 0, - "in_create": 1, - "is_submittable": 0, - "issingle": 1, - "istable": 0, - "max_attachments": 0, - "modified": "2019-02-04 23:10:30.136807", - "modified_by": "Administrator", - "module": "Accounts", - "name": "Chart of Accounts Importer", - "name_case": "", - "owner": "Administrator", + ], + "hide_toolbar": 1, + "in_create": 1, + "issingle": 1, + "links": [], + "modified": "2020-02-22 19:32:33.349075", + "modified_by": "Administrator", + "module": "Accounts", + "name": "Chart of Accounts Importer", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "cancel": 0, - "create": 1, - "delete": 0, - "email": 0, - "export": 0, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 0, - "read": 1, - "report": 0, - "role": "Accounts Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, + "create": 1, + "read": 1, + "role": "Accounts Manager", + "share": 1, "write": 1 } - ], - "quick_entry": 1, - "read_only": 1, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "", - "sort_order": "DESC", - "track_changes": 0, - "track_seen": 0, - "track_views": 0 + ], + "quick_entry": 1, + "read_only": 1, + "sort_order": "DESC" } \ No newline at end of file diff --git a/erpnext/accounts/doctype/chart_of_accounts_importer/chart_of_accounts_importer.py b/erpnext/accounts/doctype/chart_of_accounts_importer/chart_of_accounts_importer.py index f496280317a..18fb75a24cd 100644 --- a/erpnext/accounts/doctype/chart_of_accounts_importer/chart_of_accounts_importer.py +++ b/erpnext/accounts/doctype/chart_of_accounts_importer/chart_of_accounts_importer.py @@ -4,18 +4,28 @@ from __future__ import unicode_literals from functools import reduce -import frappe, csv +import frappe, csv, os from frappe import _ -from frappe.utils import cstr +from frappe.utils import cstr, cint from frappe.model.document import Document from frappe.utils.csvutils import UnicodeWriter from erpnext.accounts.doctype.account.chart_of_accounts.chart_of_accounts import create_charts, build_tree_from_json +from frappe.utils.xlsxutils import read_xlsx_file_from_attached_file, read_xls_file_from_attached_file class ChartofAccountsImporter(Document): pass @frappe.whitelist() def validate_company(company): + parent_company, allow_account_creation_against_child_company = frappe.db.get_value('Company', + {'name': company}, ['parent_company', + 'allow_account_creation_against_child_company']) + + if parent_company and (not allow_account_creation_against_child_company): + frappe.throw(_("""{0} is child company. Please import accounts in parent company + or enable {1} in company master""").format(frappe.bold(company), + frappe.bold('Allow Account Creation Against Child Company'))) + if frappe.db.get_all('GL Entry', {"company": company}, "name", limit=1): return False @@ -25,42 +35,85 @@ def import_coa(file_name, company): unset_existing_data(company) # create accounts - forest = build_forest(generate_data_from_csv(file_name)) + file_doc, extension = get_file(file_name) + + if extension == 'csv': + data = generate_data_from_csv(file_doc) + else: + data = generate_data_from_excel(file_doc, extension) + + forest = build_forest(data) create_charts(company, custom_chart=forest) # trigger on_update for company to reset default accounts set_default_accounts(company) -def generate_data_from_csv(file_name, as_dict=False): - ''' read csv file and return the generated nested tree ''' - if not file_name.endswith('.csv'): - frappe.throw("Only CSV files can be used to for importing data. Please check the file format you are trying to upload") +def get_file(file_name): + file_doc = frappe.get_doc("File", {"file_url": file_name}) + parts = file_doc.get_extension() + extension = parts[1] + extension = extension.lstrip(".") + + if extension not in ('csv', 'xlsx', 'xls'): + frappe.throw("Only CSV and Excel files can be used to for importing data. Please check the file format you are trying to upload") + + return file_doc, extension + +def generate_data_from_csv(file_doc, as_dict=False): + ''' read csv file and return the generated nested tree ''' - file_doc = frappe.get_doc('File', {"file_url": file_name}) file_path = file_doc.get_full_path() data = [] with open(file_path, 'r') as in_file: csv_reader = list(csv.reader(in_file)) - headers = csv_reader[1][1:] - del csv_reader[0:2] # delete top row and headers row + headers = csv_reader[0] + del csv_reader[0] # delete top row and headers row for row in csv_reader: if as_dict: - data.append({frappe.scrub(header): row[index+1] for index, header in enumerate(headers)}) + data.append({frappe.scrub(header): row[index] for index, header in enumerate(headers)}) else: - if not row[2]: row[2] = row[1] - data.append(row[1:]) + if not row[1]: row[1] = row[0] + data.append(row) # convert csv data return data +def generate_data_from_excel(file_doc, extension, as_dict=False): + content = file_doc.get_content() + + if extension == "xlsx": + rows = read_xlsx_file_from_attached_file(fcontent=content) + elif extension == "xls": + rows = read_xls_file_from_attached_file(content) + + data = [] + headers = rows[0] + del rows[0] + + for row in rows: + if as_dict: + data.append({frappe.scrub(header): row[index] for index, header in enumerate(headers)}) + else: + if not row[1]: row[1] = row[0] + data.append(row) + + return data + @frappe.whitelist() -def get_coa(doctype, parent, is_root=False, file_name=None): +def get_coa(doctype, parent, file_type, is_root=False, file_name=None): ''' called by tree view (to fetch node's children) ''' + file_doc, extension = get_file(file_name) parent = None if parent==_('All Accounts') else parent - forest = build_forest(generate_data_from_csv(file_name)) + + if extension == 'csv': + data = generate_data_from_csv(file_doc) + else: + data = generate_data_from_excel(file_doc, extension) + + forest = build_forest(data) accounts = build_tree_from_json("", chart_data=forest) # returns alist of dict in a tree render-able form # filter out to show data for the selected node only @@ -114,7 +167,7 @@ def build_forest(data): error_messages.append("Row {0}: Please enter Account Name".format(line_no)) charts_map[account_name] = {} - if is_group == 1: charts_map[account_name]["is_group"] = is_group + if cint(is_group) == 1: charts_map[account_name]["is_group"] = is_group if account_type: charts_map[account_name]["account_type"] = account_type if root_type: charts_map[account_name]["root_type"] = root_type if account_number: charts_map[account_name]["account_number"] = account_number @@ -132,24 +185,59 @@ def build_forest(data): return out +def build_response_as_excel(writer): + filename = frappe.generate_hash("", 10) + with open(filename, 'wb') as f: + f.write(cstr(writer.getvalue()).encode('utf-8')) + f = open(filename) + reader = csv.reader(f) + + from frappe.utils.xlsxutils import make_xlsx + xlsx_file = make_xlsx(reader, "Chart Of Accounts Importer Template") + + f.close() + os.remove(filename) + + # write out response as a xlsx type + frappe.response['filename'] = 'coa_importer_template.xlsx' + frappe.response['filecontent'] = xlsx_file.getvalue() + frappe.response['type'] = 'binary' + @frappe.whitelist() -def download_template(): +def download_template(file_type): data = frappe._dict(frappe.local.form_dict) + fields = ["Account Name", "Parent Account", "Account Number", "Is Group", "Account Type", "Root Type"] writer = UnicodeWriter() - writer.writerow([_('Chart of Accounts Template')]) - writer.writerow([_("Column Labels : ")] + fields) - writer.writerow([_("Start entering data from here : ")]) + writer.writerow(fields) - # download csv file - frappe.response['result'] = cstr(writer.getvalue()) - frappe.response['type'] = 'csv' - frappe.response['doctype'] = data.get('doctype') + for root_type in get_root_types(): + writer.writerow(['', '', '', 1, '', root_type]) + + for account in get_mandatory_group_accounts(): + writer.writerow(['', '', '', 1, account, 'Asset']) + + for account_type in get_mandatory_account_types(): + writer.writerow(['', '', '', 0, account_type.get('account_type'), account_type.get('root_type')]) + + if file_type == 'CSV': + # download csv file + frappe.response['result'] = cstr(writer.getvalue()) + frappe.response['type'] = 'csv' + frappe.response['doctype'] = 'Chart of Accounts Importer' + else: + build_response_as_excel(writer) @frappe.whitelist() def validate_accounts(file_name): - accounts = generate_data_from_csv(file_name, as_dict=True) + + file_doc, extension = get_file(file_name) + + if extension == 'csv': + accounts = generate_data_from_csv(file_doc, as_dict=True) + else: + accounts = generate_data_from_excel(file_doc, extension, as_dict=True) accounts_dict = {} for account in accounts: @@ -174,12 +262,38 @@ def validate_root(accounts): for account in roots: if not account.get("root_type") and account.get("account_name"): error_messages.append("Please enter Root Type for account- {0}".format(account.get("account_name"))) - elif account.get("root_type") not in ("Asset", "Liability", "Expense", "Income", "Equity") and account.get("account_name"): + elif account.get("root_type") not in get_root_types() and account.get("account_name"): error_messages.append("Root Type for {0} must be one of the Asset, Liability, Income, Expense and Equity".format(account.get("account_name"))) if error_messages: return "
".join(error_messages) +def get_root_types(): + return ('Asset', 'Liability', 'Expense', 'Income', 'Equity') + +def get_report_type(root_type): + if root_type in ('Asset', 'Liability', 'Equity'): + return 'Balance Sheet' + else: + return 'Profit and Loss' + +def get_mandatory_group_accounts(): + return ('Bank', 'Cash', 'Stock') + +def get_mandatory_account_types(): + return [ + {'account_type': 'Cost of Goods Sold', 'root_type': 'Expense'}, + {'account_type': 'Depreciation', 'root_type': 'Expense'}, + {'account_type': 'Fixed Asset', 'root_type': 'Asset'}, + {'account_type': 'Payable', 'root_type': 'Liability'}, + {'account_type': 'Receivable', 'root_type': 'Asset'}, + {'account_type': 'Stock Adjustment', 'root_type': 'Expense'}, + {'account_type': 'Bank', 'root_type': 'Asset'}, + {'account_type': 'Cash', 'root_type': 'Asset'}, + {'account_type': 'Stock', 'root_type': 'Asset'} + ] + + def validate_account_types(accounts): account_types_for_ledger = ["Cost of Goods Sold", "Depreciation", "Fixed Asset", "Payable", "Receivable", "Stock Adjustment"] account_types = [accounts[d]["account_type"] for d in accounts if not accounts[d]['is_group'] == 1]