[19.0][IMP] queue_job: support Odoo.sh

queue_job/README.rst

@@ -137,6 +137,13 @@ Configuration
     - ``ODOO_QUEUE_JOB_HTTP_AUTH_PASSWORD=s3cr3t``, default empty
     - Start Odoo with ``--load=web,queue_job`` and ``--workers`` greater
       than 1. [1]_
+    - On Odoo.sh, if no explicit queue job host is configured and
+      ``ODOO_STAGE`` is present, the runner derives the host from the
+      database name: ``<db_name>.dev.odoo.com`` for non-production stages
+      and ``<db_name>.odoo.com`` for production.
+    - That canonical Odoo.sh hostname must remain reachable from the Odoo
+      workers. If your setup must use another host, set
+      ``ODOO_QUEUE_JOB_HOST`` explicitly.
 
 - Using the Odoo configuration file:
 
@@ -716,6 +723,7 @@ Contributors
 - Nguyen Minh Chien <chien@trobz.com>
 - Tran Quoc Duong <duongtq@trobz.com>
 - Vo Hong Thien <thienvh@trobz.com>
+- Youssef Egla <youssefegla@gmail.com>
 
 Other credits
 -------------

queue_job/jobrunner/__init__.py

@@ -3,6 +3,7 @@
 # License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl.html)
 
 import logging
+import os
 from threading import Thread
 import time
 from configparser import ConfigParser
@@ -100,6 +101,40 @@ def _is_runner_enabled():
     return not _channels().strip().startswith("root:0")
 
 
+def _should_start_runner_thread_lazily(
+    *,
+    stage=None,
+    stop_after_init=None,
+    http_enable=None,
+    server_wide_modules=None,
+):
+    if stage is None:
+        stage = os.environ.get("ODOO_STAGE")
+    if stop_after_init is None:
+        stop_after_init = config["stop_after_init"]
+    if http_enable is None:
+        http_enable = config["http_enable"]
+    if server_wide_modules is None:
+        server_wide_modules = config["server_wide_modules"]
+    return bool(
+        stage
+        and not stop_after_init
+        and http_enable
+        and "queue_job" not in server_wide_modules
+        and _is_runner_enabled()
+    )
+
+
+def maybe_start_runner_thread(server_type):
+    global runner_thread
+    if runner_thread or not _should_start_runner_thread_lazily():
+        return False
+    _logger.info("starting jobrunner thread (in %s)", server_type)
+    runner_thread = QueueJobRunnerThread()
+    runner_thread.start()
+    return True
+
+
 def _start_runner_thread(server_type):
     global runner_thread
     if not config["stop_after_init"]:

queue_job/jobrunner/runner.py

@@ -71,6 +71,34 @@ def _odoo_now():
     return time.time()
 
 
+def _odoo_sh_stage():
+    return os.environ.get("ODOO_STAGE")
+
+
+def _odoo_sh_host(db_name, stage=None):
+    stage = stage or _odoo_sh_stage()
+    if not stage:
+        return None
+    if stage == "production":
+        return f"{db_name}.odoo.com"
+    return f"{db_name}.dev.odoo.com"
+
+
+def _jobrunner_target(db_name, scheme=None, host=None, port=None):
+    stage = _odoo_sh_stage()
+    if stage:
+        return (
+            scheme or "https",
+            host or _odoo_sh_host(db_name, stage=stage),
+            port or 443,
+        )
+    return (
+        scheme or "http",
+        host or config["http_interface"] or "localhost",
+        port or config["http_port"] or 8069,
+    )
+
+
 def _connection_info_for(db_name):
     db_or_uri, connection_info = odoo.sql_db.connection_info_for(db_name)
 
@@ -315,9 +343,9 @@ def requeue_dead_jobs(self):
 class QueueJobRunner:
     def __init__(
         self,
-        scheme="http",
-        host="localhost",
-        port=8069,
+        scheme=None,
+        host=None,
+        port=None,
         user=None,
         password=None,
         channel_config_string=None,
@@ -351,31 +379,31 @@ def from_environ_or_config(cls):
         scheme = os.environ.get("ODOO_QUEUE_JOB_SCHEME") or queue_job_config.get(
             "scheme"
         )
-        host = (
-            os.environ.get("ODOO_QUEUE_JOB_HOST")
-            or queue_job_config.get("host")
-            or config["http_interface"]
-        )
-        port = (
-            os.environ.get("ODOO_QUEUE_JOB_PORT")
-            or queue_job_config.get("port")
-            or config["http_port"]
-        )
+        host = os.environ.get("ODOO_QUEUE_JOB_HOST") or queue_job_config.get("host")
+        port = os.environ.get("ODOO_QUEUE_JOB_PORT") or queue_job_config.get("port")
         user = os.environ.get("ODOO_QUEUE_JOB_HTTP_AUTH_USER") or queue_job_config.get(
             "http_auth_user"
         )
         password = os.environ.get(
             "ODOO_QUEUE_JOB_HTTP_AUTH_PASSWORD"
         ) or queue_job_config.get("http_auth_password")
         runner = cls(
-            scheme=scheme or "http",
-            host=host or "localhost",
-            port=port or 8069,
+            scheme=scheme,
+            host=host,
+            port=port,
             user=user,
             password=password,
         )
         return runner
 
+    def _target_for_db(self, db_name):
+        return _jobrunner_target(
+            db_name,
+            scheme=self.scheme,
+            host=self.host,
+            port=self.port,
+        )
+
     def get_db_names(self):
         db_names = config["db_name"]
         if db_names:
@@ -417,10 +445,11 @@ def run_jobs(self):
                 break
             _logger.info("asking Odoo to run job %s on db %s", job.uuid, job.db_name)
             self.db_by_name[job.db_name].set_job_enqueued(job.uuid)
+            scheme, host, port = self._target_for_db(job.db_name)
             _async_http_get(
-                self.scheme,
-                self.host,
-                self.port,
+                scheme,
+                host,
+                port,
                 self.user,
                 self.password,
                 job.db_name,

queue_job/post_load.py

@@ -2,6 +2,8 @@
 
 from odoo import http
 
+from .jobrunner import maybe_start_runner_thread
+
 _logger = logging.getLogger(__name__)
 
 
@@ -11,6 +13,7 @@ def post_load():
         " from request with multiple databases"
     )
     _get_session_and_dbname_orig = http.Request._get_session_and_dbname
+    _serve_db_orig = http.Request._serve_db
 
     def _get_session_and_dbname(self):
         session, dbname = _get_session_and_dbname_orig(self)
@@ -22,4 +25,10 @@ def _get_session_and_dbname(self):
             dbname = self.httprequest.args["db"]
         return session, dbname
 
+    def _serve_db(self):
+        maybe_start_runner_thread("lazy http worker")
+        return _serve_db_orig(self)
+
     http.Request._get_session_and_dbname = _get_session_and_dbname
+    http.Request._serve_db = _serve_db
+    maybe_start_runner_thread("current http worker")

queue_job/readme/CONFIGURE.md

@@ -8,11 +8,17 @@
       or `localhost` if unset
     - `ODOO_QUEUE_JOB_HTTP_AUTH_USER=jobrunner`, default empty
     - `ODOO_QUEUE_JOB_HTTP_AUTH_PASSWORD=s3cr3t`, default empty
-    - Start Odoo with `--load=web,queue_job` and `--workers` greater than
-      1.[^1]
+    - Start Odoo with `--load=web,queue_job` and `--workers` greater than 1.[^1]
+    - On Odoo.sh, if no explicit queue job host is configured and `ODOO_STAGE`
+      is present, the runner derives the host from the database name:
+      `<db_name>.dev.odoo.com` for non-production stages and
+      `<db_name>.odoo.com` for production.
+    - That canonical Odoo.sh hostname must remain reachable from the Odoo
+      workers. If your setup must use another host, set
+      `ODOO_QUEUE_JOB_HOST` explicitly.
 - Using the Odoo configuration file:
 
-``` ini
+```ini
 [options]
 (...)
 workers = 6
@@ -31,7 +37,7 @@ http_auth_password = s3cr3t
 - Confirm the runner is starting correctly by checking the odoo log
   file:
 
-``` 
+```
 ...INFO...queue_job.jobrunner.runner: starting
 ...INFO...queue_job.jobrunner.runner: initializing database connections
 ...INFO...queue_job.jobrunner.runner: queue job runner ready for db <dbname>
@@ -43,8 +49,9 @@ http_auth_password = s3cr3t
 - Tip: to enable debug logging for the queue job, use
   `--log-handler=odoo.addons.queue_job:DEBUG`
 
-[^1]: It works with the threaded Odoo server too, although this way of
+[^1]:
+    It works with the threaded Odoo server too, although this way of
     running Odoo is obviously not for production purposes.
 
-* Jobs that remain in `enqueued` or `started` state (because, for instance,
+- Jobs that remain in `enqueued` or `started` state (because, for instance,
   their worker has been killed) will be automatically re-queued.

queue_job/readme/CONTRIBUTORS.md

@@ -13,3 +13,4 @@
 - Nguyen Minh Chien \<<chien@trobz.com>\>
 - Tran Quoc Duong \<<duongtq@trobz.com>>
 - Vo Hong Thien \<<thienvh@trobz.com>>
+- Youssef Egla \<<youssefegla@gmail.com>\>

queue_job/static/description/index.html

@@ -497,6 +497,9 @@ <h2><a class="toc-backref" href="#toc-entry-3">Configuration</a></h2>
 <li><tt class="docutils literal">ODOO_QUEUE_JOB_HTTP_AUTH_PASSWORD=s3cr3t</tt>, default empty</li>
 <li>Start Odoo with <tt class="docutils literal"><span class="pre">--load=web,queue_job</span></tt> and <tt class="docutils literal"><span class="pre">--workers</span></tt> greater
 than 1. <a class="footnote-reference" href="#footnote-1" id="footnote-reference-1">[1]</a></li>
+<li>That canonical Odoo.sh hostname must remain reachable from the Odoo
+workers. If your setup must use another host, set
+<tt class="docutils literal"><span class="pre">ODOO_QUEUE_JOB_HOST</span></tt> explicitly.</li>
 </ul>
 </li>
 </ul>
@@ -1017,6 +1020,7 @@ <h3><a class="toc-backref" href="#toc-entry-18">Contributors</a></h3>
 <li>Nguyen Minh Chien &lt;<a class="reference external" href="mailto:chien&#64;trobz.com">chien&#64;trobz.com</a>&gt;</li>
 <li>Tran Quoc Duong &lt;<a class="reference external" href="mailto:duongtq&#64;trobz.com">duongtq&#64;trobz.com</a>&gt;</li>
 <li>Vo Hong Thien &lt;<a class="reference external" href="mailto:thienvh&#64;trobz.com">thienvh&#64;trobz.com</a>&gt;</li>
+<li>Youssef Egla &lt;<a class="reference external" href="mailto:youssefegla&#64;gmail.com">youssefegla&#64;gmail.com</a>&gt;</li>
 </ul>
 </div>
 <div class="section" id="other-credits">

queue_job/tests/test_runner_runner.py

@@ -4,9 +4,11 @@
 # pylint: disable=odoo-addons-relative-import
 # we are testing, we want to test as we were an external consumer of the API
 import os
+from unittest.mock import patch
 
 from odoo.tests import BaseCase, tagged
 
+from odoo.addons.queue_job import jobrunner as jobrunner_bootstrap
 from odoo.addons.queue_job.jobrunner import runner
 
 from .common import load_doctests
@@ -57,3 +59,89 @@ def test_runner_file_closed_write_descriptor(self):
 
         self.assertFalse(self._is_open_file_descriptor(read_fd))
         self.assertFalse(self._is_open_file_descriptor(write_fd))
+
+    def test_jobrunner_target_uses_odoo_sh_dev_domain(self):
+        with patch.dict(os.environ, {"ODOO_STAGE": "dev"}, clear=False):
+            self.assertEqual(
+                ("https", "jb-web-feat-jb-paddle-31042512.dev.odoo.com", 443),
+                runner._jobrunner_target("jb-web-feat-jb-paddle-31042512"),
+            )
+
+    def test_jobrunner_target_uses_odoo_sh_staging_domain(self):
+        with patch.dict(os.environ, {"ODOO_STAGE": "staging"}, clear=False):
+            self.assertEqual(
+                (
+                    "https",
+                    "example-staging-db-12345678.dev.odoo.com",
+                    443,
+                ),
+                runner._jobrunner_target("example-staging-db-12345678"),
+            )
+
+    def test_jobrunner_target_uses_odoo_sh_production_domain(self):
+        with patch.dict(os.environ, {"ODOO_STAGE": "production"}, clear=False):
+            self.assertEqual(
+                ("https", "jb-web.odoo.com", 443),
+                runner._jobrunner_target("jb-web"),
+            )
+
+    def test_jobrunner_target_prefers_explicit_values_on_odoo_sh(self):
+        with patch.dict(os.environ, {"ODOO_STAGE": "staging"}, clear=False):
+            self.assertEqual(
+                ("https", "custom.example.com", 8443),
+                runner._jobrunner_target(
+                    "example-staging-db-12345678",
+                    scheme="https",
+                    host="custom.example.com",
+                    port=8443,
+                ),
+            )
+
+    def test_should_start_runner_thread_lazily_on_odoosh_http_process(self):
+        with patch.object(jobrunner_bootstrap, "_is_runner_enabled", return_value=True):
+            self.assertTrue(
+                jobrunner_bootstrap._should_start_runner_thread_lazily(
+                    stage="staging",
+                    stop_after_init=False,
+                    http_enable=True,
+                    server_wide_modules=["base", "web"],
+                )
+            )
+
+    def test_should_not_start_runner_thread_lazily_when_server_wide(self):
+        with patch.object(jobrunner_bootstrap, "_is_runner_enabled", return_value=True):
+            self.assertFalse(
+                jobrunner_bootstrap._should_start_runner_thread_lazily(
+                    stage="production",
+                    stop_after_init=False,
+                    http_enable=True,
+                    server_wide_modules=["base", "web", "queue_job"],
+                )
+            )
+
+    def test_maybe_start_runner_thread_starts_only_once(self):
+        original_runner_thread = jobrunner_bootstrap.runner_thread
+        try:
+            jobrunner_bootstrap.runner_thread = None
+            fake_thread = type("FakeThread", (), {"start": lambda self: None})()
+            with (
+                patch.object(
+                    jobrunner_bootstrap,
+                    "_should_start_runner_thread_lazily",
+                    return_value=True,
+                ),
+                patch.object(
+                    jobrunner_bootstrap,
+                    "QueueJobRunnerThread",
+                    return_value=fake_thread,
+                ) as thread_cls,
+            ):
+                self.assertTrue(
+                    jobrunner_bootstrap.maybe_start_runner_thread("lazy http worker")
+                )
+                self.assertFalse(
+                    jobrunner_bootstrap.maybe_start_runner_thread("lazy http worker")
+                )
+                self.assertEqual(1, thread_cls.call_count)
+        finally:
+            jobrunner_bootstrap.runner_thread = original_runner_thread