diff --git a/queue_job/README.rst b/queue_job/README.rst index 634f2d510..088c47a8d 100644 --- a/queue_job/README.rst +++ b/queue_job/README.rst @@ -137,6 +137,13 @@ Configuration - ``ODOO_QUEUE_JOB_HTTP_AUTH_PASSWORD=s3cr3t``, default empty - Start Odoo with ``--load=web,queue_job`` and ``--workers`` greater than 1. [1]_ + - On Odoo.sh, if no explicit queue job host is configured and + ``ODOO_STAGE`` is present, the runner derives the host from the + database name: ``.dev.odoo.com`` for non-production stages + and ``.odoo.com`` for production. + - That canonical Odoo.sh hostname must remain reachable from the Odoo + workers. If your setup must use another host, set + ``ODOO_QUEUE_JOB_HOST`` explicitly. - Using the Odoo configuration file: @@ -716,6 +723,7 @@ Contributors - Nguyen Minh Chien - Tran Quoc Duong - Vo Hong Thien +- Youssef Egla Other credits ------------- diff --git a/queue_job/jobrunner/__init__.py b/queue_job/jobrunner/__init__.py index 50dd45e39..7fd907bf0 100644 --- a/queue_job/jobrunner/__init__.py +++ b/queue_job/jobrunner/__init__.py @@ -3,6 +3,7 @@ # License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl.html) import logging +import os from threading import Thread import time from configparser import ConfigParser @@ -100,6 +101,40 @@ def _is_runner_enabled(): return not _channels().strip().startswith("root:0") +def _should_start_runner_thread_lazily( + *, + stage=None, + stop_after_init=None, + http_enable=None, + server_wide_modules=None, +): + if stage is None: + stage = os.environ.get("ODOO_STAGE") + if stop_after_init is None: + stop_after_init = config["stop_after_init"] + if http_enable is None: + http_enable = config["http_enable"] + if server_wide_modules is None: + server_wide_modules = config["server_wide_modules"] + return bool( + stage + and not stop_after_init + and http_enable + and "queue_job" not in server_wide_modules + and _is_runner_enabled() + ) + + +def maybe_start_runner_thread(server_type): + global runner_thread + if runner_thread or not _should_start_runner_thread_lazily(): + return False + _logger.info("starting jobrunner thread (in %s)", server_type) + runner_thread = QueueJobRunnerThread() + runner_thread.start() + return True + + def _start_runner_thread(server_type): global runner_thread if not config["stop_after_init"]: diff --git a/queue_job/jobrunner/runner.py b/queue_job/jobrunner/runner.py index 7fd91d68b..1a8948928 100644 --- a/queue_job/jobrunner/runner.py +++ b/queue_job/jobrunner/runner.py @@ -71,6 +71,34 @@ def _odoo_now(): return time.time() +def _odoo_sh_stage(): + return os.environ.get("ODOO_STAGE") + + +def _odoo_sh_host(db_name, stage=None): + stage = stage or _odoo_sh_stage() + if not stage: + return None + if stage == "production": + return f"{db_name}.odoo.com" + return f"{db_name}.dev.odoo.com" + + +def _jobrunner_target(db_name, scheme=None, host=None, port=None): + stage = _odoo_sh_stage() + if stage: + return ( + scheme or "https", + host or _odoo_sh_host(db_name, stage=stage), + port or 443, + ) + return ( + scheme or "http", + host or config["http_interface"] or "localhost", + port or config["http_port"] or 8069, + ) + + def _connection_info_for(db_name): db_or_uri, connection_info = odoo.sql_db.connection_info_for(db_name) @@ -315,9 +343,9 @@ def requeue_dead_jobs(self): class QueueJobRunner: def __init__( self, - scheme="http", - host="localhost", - port=8069, + scheme=None, + host=None, + port=None, user=None, password=None, channel_config_string=None, @@ -351,16 +379,8 @@ def from_environ_or_config(cls): scheme = os.environ.get("ODOO_QUEUE_JOB_SCHEME") or queue_job_config.get( "scheme" ) - host = ( - os.environ.get("ODOO_QUEUE_JOB_HOST") - or queue_job_config.get("host") - or config["http_interface"] - ) - port = ( - os.environ.get("ODOO_QUEUE_JOB_PORT") - or queue_job_config.get("port") - or config["http_port"] - ) + host = os.environ.get("ODOO_QUEUE_JOB_HOST") or queue_job_config.get("host") + port = os.environ.get("ODOO_QUEUE_JOB_PORT") or queue_job_config.get("port") user = os.environ.get("ODOO_QUEUE_JOB_HTTP_AUTH_USER") or queue_job_config.get( "http_auth_user" ) @@ -368,14 +388,22 @@ def from_environ_or_config(cls): "ODOO_QUEUE_JOB_HTTP_AUTH_PASSWORD" ) or queue_job_config.get("http_auth_password") runner = cls( - scheme=scheme or "http", - host=host or "localhost", - port=port or 8069, + scheme=scheme, + host=host, + port=port, user=user, password=password, ) return runner + def _target_for_db(self, db_name): + return _jobrunner_target( + db_name, + scheme=self.scheme, + host=self.host, + port=self.port, + ) + def get_db_names(self): db_names = config["db_name"] if db_names: @@ -417,10 +445,11 @@ def run_jobs(self): break _logger.info("asking Odoo to run job %s on db %s", job.uuid, job.db_name) self.db_by_name[job.db_name].set_job_enqueued(job.uuid) + scheme, host, port = self._target_for_db(job.db_name) _async_http_get( - self.scheme, - self.host, - self.port, + scheme, + host, + port, self.user, self.password, job.db_name, diff --git a/queue_job/post_load.py b/queue_job/post_load.py index f0c1df870..76440cafe 100644 --- a/queue_job/post_load.py +++ b/queue_job/post_load.py @@ -2,6 +2,8 @@ from odoo import http +from .jobrunner import maybe_start_runner_thread + _logger = logging.getLogger(__name__) @@ -11,6 +13,7 @@ def post_load(): " from request with multiple databases" ) _get_session_and_dbname_orig = http.Request._get_session_and_dbname + _serve_db_orig = http.Request._serve_db def _get_session_and_dbname(self): session, dbname = _get_session_and_dbname_orig(self) @@ -22,4 +25,10 @@ def _get_session_and_dbname(self): dbname = self.httprequest.args["db"] return session, dbname + def _serve_db(self): + maybe_start_runner_thread("lazy http worker") + return _serve_db_orig(self) + http.Request._get_session_and_dbname = _get_session_and_dbname + http.Request._serve_db = _serve_db + maybe_start_runner_thread("current http worker") diff --git a/queue_job/readme/CONFIGURE.md b/queue_job/readme/CONFIGURE.md index 723910621..e3021d95a 100644 --- a/queue_job/readme/CONFIGURE.md +++ b/queue_job/readme/CONFIGURE.md @@ -8,11 +8,17 @@ or `localhost` if unset - `ODOO_QUEUE_JOB_HTTP_AUTH_USER=jobrunner`, default empty - `ODOO_QUEUE_JOB_HTTP_AUTH_PASSWORD=s3cr3t`, default empty - - Start Odoo with `--load=web,queue_job` and `--workers` greater than - 1.[^1] + - Start Odoo with `--load=web,queue_job` and `--workers` greater than 1.[^1] + - On Odoo.sh, if no explicit queue job host is configured and `ODOO_STAGE` + is present, the runner derives the host from the database name: + `.dev.odoo.com` for non-production stages and + `.odoo.com` for production. + - That canonical Odoo.sh hostname must remain reachable from the Odoo + workers. If your setup must use another host, set + `ODOO_QUEUE_JOB_HOST` explicitly. - Using the Odoo configuration file: -``` ini +```ini [options] (...) workers = 6 @@ -31,7 +37,7 @@ http_auth_password = s3cr3t - Confirm the runner is starting correctly by checking the odoo log file: -``` +``` ...INFO...queue_job.jobrunner.runner: starting ...INFO...queue_job.jobrunner.runner: initializing database connections ...INFO...queue_job.jobrunner.runner: queue job runner ready for db @@ -43,8 +49,9 @@ http_auth_password = s3cr3t - Tip: to enable debug logging for the queue job, use `--log-handler=odoo.addons.queue_job:DEBUG` -[^1]: It works with the threaded Odoo server too, although this way of +[^1]: + It works with the threaded Odoo server too, although this way of running Odoo is obviously not for production purposes. -* Jobs that remain in `enqueued` or `started` state (because, for instance, +- Jobs that remain in `enqueued` or `started` state (because, for instance, their worker has been killed) will be automatically re-queued. diff --git a/queue_job/readme/CONTRIBUTORS.md b/queue_job/readme/CONTRIBUTORS.md index 9f92cfb5a..af4cf7ddc 100644 --- a/queue_job/readme/CONTRIBUTORS.md +++ b/queue_job/readme/CONTRIBUTORS.md @@ -13,3 +13,4 @@ - Nguyen Minh Chien \<\> - Tran Quoc Duong \<> - Vo Hong Thien \<> +- Youssef Egla \<\> diff --git a/queue_job/static/description/index.html b/queue_job/static/description/index.html index 6a95db1d6..1f3e23aea 100644 --- a/queue_job/static/description/index.html +++ b/queue_job/static/description/index.html @@ -497,6 +497,9 @@

Configuration

  • ODOO_QUEUE_JOB_HTTP_AUTH_PASSWORD=s3cr3t, default empty
  • Start Odoo with --load=web,queue_job and --workers greater than 1. [1]
    • +
  • That canonical Odoo.sh hostname must remain reachable from the Odoo +workers. If your setup must use another host, set +ODOO_QUEUE_JOB_HOST explicitly.
    • @@ -1017,6 +1020,7 @@

    Contributors

  • Nguyen Minh Chien <chien@trobz.com>
  • Tran Quoc Duong <duongtq@trobz.com>
  • Vo Hong Thien <thienvh@trobz.com>
    • +
  • Youssef Egla <youssefegla@gmail.com>
    • diff --git a/queue_job/tests/test_runner_runner.py b/queue_job/tests/test_runner_runner.py index 131ce6322..d909e486a 100644 --- a/queue_job/tests/test_runner_runner.py +++ b/queue_job/tests/test_runner_runner.py @@ -4,9 +4,11 @@ # pylint: disable=odoo-addons-relative-import # we are testing, we want to test as we were an external consumer of the API import os +from unittest.mock import patch from odoo.tests import BaseCase, tagged +from odoo.addons.queue_job import jobrunner as jobrunner_bootstrap from odoo.addons.queue_job.jobrunner import runner from .common import load_doctests @@ -57,3 +59,89 @@ def test_runner_file_closed_write_descriptor(self): self.assertFalse(self._is_open_file_descriptor(read_fd)) self.assertFalse(self._is_open_file_descriptor(write_fd)) + + def test_jobrunner_target_uses_odoo_sh_dev_domain(self): + with patch.dict(os.environ, {"ODOO_STAGE": "dev"}, clear=False): + self.assertEqual( + ("https", "jb-web-feat-jb-paddle-31042512.dev.odoo.com", 443), + runner._jobrunner_target("jb-web-feat-jb-paddle-31042512"), + ) + + def test_jobrunner_target_uses_odoo_sh_staging_domain(self): + with patch.dict(os.environ, {"ODOO_STAGE": "staging"}, clear=False): + self.assertEqual( + ( + "https", + "example-staging-db-12345678.dev.odoo.com", + 443, + ), + runner._jobrunner_target("example-staging-db-12345678"), + ) + + def test_jobrunner_target_uses_odoo_sh_production_domain(self): + with patch.dict(os.environ, {"ODOO_STAGE": "production"}, clear=False): + self.assertEqual( + ("https", "jb-web.odoo.com", 443), + runner._jobrunner_target("jb-web"), + ) + + def test_jobrunner_target_prefers_explicit_values_on_odoo_sh(self): + with patch.dict(os.environ, {"ODOO_STAGE": "staging"}, clear=False): + self.assertEqual( + ("https", "custom.example.com", 8443), + runner._jobrunner_target( + "example-staging-db-12345678", + scheme="https", + host="custom.example.com", + port=8443, + ), + ) + + def test_should_start_runner_thread_lazily_on_odoosh_http_process(self): + with patch.object(jobrunner_bootstrap, "_is_runner_enabled", return_value=True): + self.assertTrue( + jobrunner_bootstrap._should_start_runner_thread_lazily( + stage="staging", + stop_after_init=False, + http_enable=True, + server_wide_modules=["base", "web"], + ) + ) + + def test_should_not_start_runner_thread_lazily_when_server_wide(self): + with patch.object(jobrunner_bootstrap, "_is_runner_enabled", return_value=True): + self.assertFalse( + jobrunner_bootstrap._should_start_runner_thread_lazily( + stage="production", + stop_after_init=False, + http_enable=True, + server_wide_modules=["base", "web", "queue_job"], + ) + ) + + def test_maybe_start_runner_thread_starts_only_once(self): + original_runner_thread = jobrunner_bootstrap.runner_thread + try: + jobrunner_bootstrap.runner_thread = None + fake_thread = type("FakeThread", (), {"start": lambda self: None})() + with ( + patch.object( + jobrunner_bootstrap, + "_should_start_runner_thread_lazily", + return_value=True, + ), + patch.object( + jobrunner_bootstrap, + "QueueJobRunnerThread", + return_value=fake_thread, + ) as thread_cls, + ): + self.assertTrue( + jobrunner_bootstrap.maybe_start_runner_thread("lazy http worker") + ) + self.assertFalse( + jobrunner_bootstrap.maybe_start_runner_thread("lazy http worker") + ) + self.assertEqual(1, thread_cls.call_count) + finally: + jobrunner_bootstrap.runner_thread = original_runner_thread