diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4cefd00cac..af6b972525 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,6 @@ exclude: | (?x) # NOT INSTALLABLE ADDONS - ^base_import_async/| ^queue_job_batch/| ^queue_job_cron/| ^queue_job_cron_jobrunner/| diff --git a/base_import_async/README.rst b/base_import_async/README.rst index 030c371758..62b734c9af 100644 --- a/base_import_async/README.rst +++ b/base_import_async/README.rst @@ -1,3 +1,7 @@ +.. image:: https://odoo-community.org/readme-banner-image + :target: https://odoo-community.org/get-involved?utm_source=readme + :alt: Odoo Community Association + =================== Asynchronous Import =================== @@ -13,17 +17,17 @@ Asynchronous Import .. |badge1| image:: https://img.shields.io/badge/maturity-Production%2FStable-green.png :target: https://odoo-community.org/page/development-status :alt: Production/Stable -.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png +.. |badge2| image:: https://img.shields.io/badge/license-AGPL--3-blue.png :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html :alt: License: AGPL-3 .. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fqueue-lightgray.png?logo=github - :target: https://github.com/OCA/queue/tree/18.0/base_import_async + :target: https://github.com/OCA/queue/tree/19.0/base_import_async :alt: OCA/queue .. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png - :target: https://translation.odoo-community.org/projects/queue-18-0/queue-18-0-base_import_async + :target: https://translation.odoo-community.org/projects/queue-19-0/queue-19-0-base_import_async :alt: Translate me on Weblate .. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png - :target: https://runboat.odoo-community.org/builds?repo=OCA/queue&target_branch=18.0 + :target: https://runboat.odoo-community.org/builds?repo=OCA/queue&target_branch=19.0 :alt: Try me on Runboat |badge1| |badge2| |badge3| |badge4| |badge5| @@ -87,7 +91,7 @@ Bug Tracker Bugs are tracked on `GitHub Issues `_. In case of trouble, please check there if your issue has already been reported. If you spotted it first, help us to smash it by providing a detailed and welcomed -`feedback `_. +`feedback `_. Do not contact contributors directly about support or help with technical issues. @@ -130,6 +134,8 @@ Other contributors include: - Daniel Duque (FactorLibre) +- Youssef Egla + Other credits ------------- @@ -149,6 +155,6 @@ OCA, or the Odoo Community Association, is a nonprofit organization whose mission is to support the collaborative development of Odoo features and promote its widespread use. -This module is part of the `OCA/queue `_ project on GitHub. +This module is part of the `OCA/queue `_ project on GitHub. You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/base_import_async/__init__.py b/base_import_async/__init__.py index 31660d6a96..adc6207fdd 100644 --- a/base_import_async/__init__.py +++ b/base_import_async/__init__.py @@ -1,3 +1,4 @@ # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). from . import models +from . import wizards diff --git a/base_import_async/__manifest__.py b/base_import_async/__manifest__.py index 5432d7c5ca..58dd6eb8b0 100644 --- a/base_import_async/__manifest__.py +++ b/base_import_async/__manifest__.py @@ -5,7 +5,7 @@ { "name": "Asynchronous Import", "summary": "Import CSV files in the background", - "version": "18.0.1.0.0", + "version": "19.0.1.0.0", "author": "Akretion, ACSONE SA/NV, Odoo Community Association (OCA)", "license": "AGPL-3", "website": "https://github.com/OCA/queue", @@ -20,6 +20,6 @@ "base_import_async/static/src/xml/import_data_sidepanel.xml", ], }, - "installable": False, + "installable": True, "development_status": "Production/Stable", } diff --git a/base_import_async/models/__init__.py b/base_import_async/models/__init__.py index 2af21f4102..b54b6491b9 100644 --- a/base_import_async/models/__init__.py +++ b/base_import_async/models/__init__.py @@ -1,4 +1,3 @@ # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). -from . import base_import_import from . import queue_job diff --git a/base_import_async/models/queue_job.py b/base_import_async/models/queue_job.py index b7313505f3..5960c74364 100644 --- a/base_import_async/models/queue_job.py +++ b/base_import_async/models/queue_job.py @@ -1,7 +1,7 @@ # Copyright 2017 ACSONE SA/NV # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). -from odoo import _, models +from odoo import models class QueueJob(models.Model): @@ -10,10 +10,16 @@ class QueueJob(models.Model): _inherit = "queue.job" def _related_action_attachment(self): + attachment = self.env["ir.attachment"].search( + [("res_model", "=", "queue.job"), ("res_id", "=", self.id)], + limit=1, + ) + if not attachment: + return None return { - "name": _("Attachment"), + "name": self.env._("Attachment"), "type": "ir.actions.act_window", "res_model": "ir.attachment", "view_mode": "form", - "res_id": self.kwargs.get("att_id"), + "res_id": attachment.id, } diff --git a/base_import_async/readme/CONTRIBUTORS.md b/base_import_async/readme/CONTRIBUTORS.md index 729ae17015..8f66996913 100644 --- a/base_import_async/readme/CONTRIBUTORS.md +++ b/base_import_async/readme/CONTRIBUTORS.md @@ -23,3 +23,5 @@ Other contributors include: - Do Anh Duy \<\> - Daniel Duque (FactorLibre) + +- Youssef Egla diff --git a/base_import_async/static/description/index.html b/base_import_async/static/description/index.html index 2994cf9efa..af24627627 100644 --- a/base_import_async/static/description/index.html +++ b/base_import_async/static/description/index.html @@ -3,7 +3,7 @@ -Asynchronous Import +README.rst -
-

Asynchronous Import

+
+ + +Odoo Community Association + +
+

Asynchronous Import

-

Production/Stable License: AGPL-3 OCA/queue Translate me on Weblate Try me on Runboat

+

Production/Stable License: AGPL-3 OCA/queue Translate me on Weblate Try me on Runboat

This module extends the standard CSV import functionality to import files in the background using the OCA/queue framework.

Table of contents

@@ -392,7 +397,7 @@

Asynchronous Import

-

Usage

+

Usage

The user is presented with a new checkbox in the import screen. When selected, the import is delayed in a background job.

This job in turn splits the CSV file in chunks of minimum 100 lines (or @@ -416,7 +421,7 @@

Usage

-

Known issues / Roadmap

+

Known issues / Roadmap

  • There is currently no user interface to control the chunk size, which is currently 100 by default. Should this proves to be an issue, it is @@ -425,33 +430,33 @@

    Known issues / Roadmap

-

Changelog

+

Changelog

-

13.0.1.0.0 (2019-12-20)

+

13.0.1.0.0 (2019-12-20)

  • [MIGRATION] from 12.0 branched at rev. a7f8031
-

Bug Tracker

+

Bug Tracker

Bugs are tracked on GitHub Issues. In case of trouble, please check there if your issue has already been reported. If you spotted it first, help us to smash it by providing a detailed and welcomed -feedback.

+feedback.

Do not contact contributors directly about support or help with technical issues.

-

Credits

+

Credits

-

Authors

+

Authors

  • Akretion
  • ACSONE SA/NV
-

Contributors

+

Contributors

Sébastien Beau (Akretion) authored the initial prototype.

Stéphane Bidoul (ACSONE) extended it to version 1.0 to support multi-line records, store data to import as attachments and let the user @@ -470,15 +475,16 @@

Contributors

  • Daniel Duque (FactorLibre)
    • +
  • Youssef Egla
    • -

    Other credits

    +

    Other credits

    The migration of this module from 17.0 to 18.0 was financially supported by Camptocamp

    -

    Maintainers

    +

    Maintainers

    This module is maintained by the OCA.

    Odoo Community Association @@ -486,10 +492,11 @@

    Maintainers

    OCA, or the Odoo Community Association, is a nonprofit organization whose mission is to support the collaborative development of Odoo features and promote its widespread use.

    -

    This module is part of the OCA/queue project on GitHub.

    +

    This module is part of the OCA/queue project on GitHub.

    You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

    +
    diff --git a/base_import_async/tests/test_base_import_import.py b/base_import_async/tests/test_base_import_import.py index 4a815d6707..828372a4e5 100644 --- a/base_import_async/tests/test_base_import_import.py +++ b/base_import_async/tests/test_base_import_import.py @@ -1,9 +1,16 @@ # Copyright 2024 Camptocamp # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +import base64 +import datetime +import uuid +from unittest import mock + from odoo.tests.common import RecordCapturer, TransactionCase -from ..models.base_import_import import OPT_USE_QUEUE +from odoo.addons.queue_job.tests.common import trap_jobs + +from ..wizards.base_import_import import OPT_USE_QUEUE class TestBaseImportImport(TransactionCase): @@ -11,89 +18,171 @@ class TestBaseImportImport(TransactionCase): def setUpClass(cls): super().setUpClass() cls.res_partners = cls.env["res.partner"] - cls.import_wizard = cls.env["base_import.import"] + cls.base_import = cls.env["base_import.import"] + cls.queue_job = cls.env["queue.job"] + + def _create_import_wizard(self, rows, file_name="partners.csv"): + csv_content = "\n".join(";".join(row) for row in rows) + return self.base_import.create( + { + "res_model": self.res_partners._name, + "file": csv_content.encode(), + "file_name": file_name, + "file_type": "text/csv", + } + ) + + def _get_import_preview(self, import_wizard, options): + preview = import_wizard.parse_preview(options) + self.assertIsNone(preview.get("error"), preview.get("error")) + return preview + + def _get_import_fields(self, import_wizard, options): + preview = self._get_import_preview(import_wizard, options) + return ["/".join(field_names) for field_names in preview["matches"].values()], ( + preview["headers"] or [] + ) def test_normal_import_res_partners(self): - values = [ + import_wizard = self._create_import_wizard( [ - "name", - "email", - "is_company", - ], + ["name", "email", "is_company"], + ["partner 1", "partner1@example.com", "1"], + ["partner 2", "partner2@example.com", "0"], + ] + ) + options = {"quoting": '"', "separator": ";", "has_headers": True} + import_fields, columns = self._get_import_fields(import_wizard, options) + + with RecordCapturer(self.res_partners, []) as capture: + result = import_wizard.execute_import(import_fields, columns, options) + + self.assertCountEqual(result["messages"], []) + self.assertEqual(len(capture.records), 2) + self.assertCountEqual( + capture.records.mapped("email"), [ - "partner 1", "partner1@example.com", - "1", - ], - [ - "partner 2", "partner2@example.com", - "0", ], - ] - import_vals = { - "res_model": self.res_partners._name, - "file": "\n".join([";".join(values) for values in values]), - "file_type": "text/csv", - } - self.import_wizard |= self.import_wizard.create(import_vals) - opts = {"quoting": '"', "separator": ";", "has_headers": True} - preview = self.import_wizard.parse_preview(opts) - self.assertEqual( - preview["matches"], - { - 0: ["name"], - 1: ["email"], - 2: ["is_company"], - }, ) - with RecordCapturer(self.res_partners, []) as capture: - results = self.import_wizard.execute_import( - [fnames[0] for fnames in preview["matches"].values()], - [], - opts, - ) - # if result is empty, no import error - self.assertItemsEqual(results["messages"], []) - records_created = capture.records - self.assertEqual(len(records_created), 2) - self.assertIn("partner1", records_created[0].email) - - def test_wrong_import_res_partners(self): - values = [ + + def test_async_import_schedules_and_imports_records(self): + import_wizard = self._create_import_wizard( [ - "name", - "email", - "date", # Adding date field to trigger parsing error - ], + ["name", "email", "is_company"], + ["async partner 1", "async1@example.com", "1"], + ["async partner 2", "async2@example.com", "0"], + ] + ) + options = { + "quoting": '"', + "separator": ";", + "has_headers": True, + OPT_USE_QUEUE: True, + } + import_fields, columns = self._get_import_fields(import_wizard, options) + + with trap_jobs() as trap: + result = import_wizard.execute_import(import_fields, columns, options) + self.assertEqual(result, []) + trap.assert_jobs_count(1, only=import_wizard._split_file) + + with RecordCapturer(self.res_partners, []) as capture: + trap.perform_enqueued_jobs() + trap.assert_jobs_count(1, only=import_wizard._import_one_chunk) + trap.perform_enqueued_jobs() + + self.assertEqual(len(capture.records), 2) + self.assertCountEqual( + capture.records.mapped("name"), [ - "partner 1", - "partner1@example.com", - "21-13-2024", + "async partner 1", + "async partner 2", ], + ) + + def test_async_import_uses_datetime_prevalidation(self): + import_wizard = self.base_import.create({"res_model": self.res_partners._name}) + + with trap_jobs() as trap: + with mock.patch.object( + type(import_wizard), + "_convert_import_data", + return_value=( + [ + [datetime.date(2026, 4, 17)], + ], + ["name"], + ), + ): + result = import_wizard.execute_import( + ["name"], + ["name"], + {OPT_USE_QUEUE: True}, + ) + + self.assertEqual(trap.jobs_count(), 0) + self.assertEqual(len(result["messages"]), 1) + self.assertIn( + "does not accept date/time values", result["messages"][0]["message"] + ) + + def test_async_import_applies_fallback_values(self): + import_wizard = self._create_import_wizard( [ - "partner 2", - "partner2@example.com", - "2024-13-45", + ["name", "company_type"], + ["fallback partner", "Unknown value"], ], - ] - opts = { + file_name="fallback.csv", + ) + options = { "quoting": '"', "separator": ";", "has_headers": True, - "date_format": "%Y-%m-%d", # Set specific date format OPT_USE_QUEUE: True, + "fallback_values": { + "company_type": { + "fallback_value": "person", + "field_model": "res.partner", + "field_type": "selection", + } + }, } - import_vals = { - "res_model": self.res_partners._name, - "file": "\n".join([";".join(row) for row in values]), - "file_type": "text/csv", - } - import_wizard = self.import_wizard.create(import_vals) - preview = import_wizard.parse_preview(opts) - results = import_wizard.execute_import( - [field[0] for field in preview["matches"].values()], - ["name", "email", "date"], # Include date in fields to import - opts, + import_fields, columns = self._get_import_fields(import_wizard, options) + + with trap_jobs() as trap: + result = import_wizard.execute_import(import_fields, columns, options) + self.assertEqual(result, []) + trap.perform_enqueued_jobs() + trap.perform_enqueued_jobs() + + partner = self.res_partners.search([("name", "=", "fallback partner")], limit=1) + self.assertTrue(partner) + self.assertEqual(partner.company_type, "person") + + def test_related_action_attachment_returns_linked_attachment(self): + queue_job = self.queue_job.with_context( + _job_edit_sentinel=self.queue_job.EDIT_SENTINEL + ).create( + { + "uuid": str(uuid.uuid4()), + "state": "done", + "user_id": self.env.user.id, + "company_id": self.env.company.id, + "kwargs": {}, + } + ) + attachment = self.env["ir.attachment"].create( + { + "name": "chunk.csv", + "datas": base64.b64encode(b"name\nlinked attachment"), + "res_model": "queue.job", + "res_id": queue_job.id, + } ) - self.assertTrue(any(msg["type"] == "error" for msg in results["messages"])) + + action = queue_job._related_action_attachment() + + self.assertEqual(action["res_model"], "ir.attachment") + self.assertEqual(action["res_id"], attachment.id) diff --git a/base_import_async/wizards/__init__.py b/base_import_async/wizards/__init__.py new file mode 100644 index 0000000000..695ed69ea5 --- /dev/null +++ b/base_import_async/wizards/__init__.py @@ -0,0 +1,3 @@ +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +from . import base_import_import diff --git a/base_import_async/models/base_import_import.py b/base_import_async/wizards/base_import_import.py similarity index 82% rename from base_import_async/models/base_import_import.py rename to base_import_async/wizards/base_import_import.py index 856828a593..de66ea3e38 100644 --- a/base_import_async/models/base_import_import.py +++ b/base_import_async/wizards/base_import_import.py @@ -9,7 +9,7 @@ from io import BytesIO, StringIO, TextIOWrapper from os.path import splitext -from odoo import _, api, models +from odoo import models from odoo.models import fix_import_export_id_paths from odoo.addons.base_import.models.base_import import ImportValidationError @@ -34,6 +34,7 @@ class BaseImportImport(models.TransientModel): _inherit = "base_import.import" def execute_import(self, fields, columns, options, dryrun=False): + self.ensure_one() if dryrun or not options.get(OPT_USE_QUEUE): # normal import return super().execute_import(fields, columns, options, dryrun=dryrun) @@ -41,8 +42,15 @@ def execute_import(self, fields, columns, options, dryrun=False): # asynchronous import try: data, import_fields = self._convert_import_data(fields, options) + if errors := self._parse_datetime_data(import_fields, data): + return {"messages": errors} # Parse date and float field data = self._parse_import_data(data, import_fields, options) + import_fields, data = self._handle_multi_mapping(import_fields, data) + if options.get("fallback_values"): + data = self._handle_fallback_values( + import_fields, data, options["fallback_values"] + ) except (ImportValidationError, ValueError) as e: return {"messages": [e.__dict__]} @@ -55,10 +63,11 @@ def execute_import(self, fields, columns, options, dryrun=False): translated_model_name = search_result[0][1] else: translated_model_name = self._description - description = _("Import %(model)s from file %(from_file)s") % { - "model": translated_model_name, - "from_file": self.file_name, - } + description = self.env._( + "Import %(model)s from file %(from_file)s", + model=translated_model_name, + from_file=self.file_name, + ) attachment = self._create_csv_attachment( import_fields, data, options, self.file_name ) @@ -78,7 +87,6 @@ def _link_attachment_to_job(self, delayed_job, attachment): ) attachment.write({"res_model": "queue.job", "res_id": queue_job.id}) - @api.returns("ir.attachment") def _create_csv_attachment(self, fields, data, options, file_name): # write csv f = StringIO() @@ -155,16 +163,15 @@ def _split_file( model_obj, fields, data, chunk_size ): chunk = str(priority - INIT_PRIORITY).zfill(padding) - description = _( - "Import %(model)s from file %(file_name)s - " - "#%(chunk)s - lines %(from)s to %(to)s" - ) % { - "model": translated_model_name, - "file_name": file_name, - "chunk": chunk, - "from": row_from + 1 + header_offset, - "to": row_to + 1 + header_offset, - } + description = self.env._( + "Import %(model)s from file %(file_name)s - #%(chunk)s - " + "lines %(from_line)s to %(to_line)s", + model=translated_model_name, + file_name=file_name, + chunk=chunk, + from_line=row_from + 1 + header_offset, + to_line=row_to + 1 + header_offset, + ) # create a CSV attachment and enqueue the job root, ext = splitext(file_name) attachment = self._create_csv_attachment( @@ -182,7 +189,14 @@ def _split_file( priority += 1 def _import_one_chunk(self, model_name, attachment, options): - model_obj = self.env[model_name] + load_context = { + "import_file": True, + "tracking_disable": options.get("tracking_disable"), + "name_create_enabled_fields": options.get("name_create_enabled_fields", {}), + "import_set_empty_fields": options.get("import_set_empty_fields", []), + "import_skip_records": options.get("import_skip_records", []), + } + model_obj = self.env[model_name].with_context(**load_context) fields, data = self._read_csv_attachment(attachment, options) result = model_obj.load(fields, data) error_message = [