diff --git a/AGENTS.md b/AGENTS.md index 5c385a78db..01d3c96e8c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -68,6 +68,10 @@ It consists of: Java (Truffle) + C (CPython C-API compatibility) + Python stdlib * Style / formatting `mx python-style --fix` `mx python-gate --tags style` +* Building standalones for benchmarking + - use `mx --env native-ee sforceimports && mx --env native-ee checkout-downstream compiler graal-enterprise` to get the right revisions + - use `mx -p ../graal/vm fetch-jdk -jdk-id labsjdk-ce-latest` and set JAVA_HOME as per that command's output + - use `mx --env jvm-ee-libgraal` and `mx --env native-ee` to build the JAVA and NATIVE standalone distributions ## NOTES - When searching for implementation, prefer `graalpython/com.oracle.graal.python/src/...` over vendored `lib-python` unless you are intentionally modifying upstream stdlib/tests. diff --git a/graalpython/com.oracle.graal.python.benchmarks/python/micro/jsonrpc-pipe.py b/graalpython/com.oracle.graal.python.benchmarks/python/micro/jsonrpc-pipe.py new file mode 100644 index 0000000000..83ee8cfd06 --- /dev/null +++ b/graalpython/com.oracle.graal.python.benchmarks/python/micro/jsonrpc-pipe.py @@ -0,0 +1,443 @@ +# Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from __future__ import annotations + +import argparse +import io +import json +import os +import re +import subprocess +import sys +import time + + +EMAIL_RE = re.compile(r"\s+") +NON_DIGIT_RE = re.compile(r"\D+") +_STATE = None + + +class Endpoint: + def __init__(self, mode, reader, writer, closeables=()): + self.mode = mode + self.reader = reader + self.writer = writer + self.closeables = closeables + + def write_message(self, message): + line = json.dumps(message, separators=(",", ":")) + if self.mode == "text": + self.writer.write(line) + self.writer.write("\n") + self.writer.flush() + return + payload = (line + "\n").encode("utf-8") + if self.mode == "buffer": + self.writer.write(payload) + self.writer.flush() + return + write_all(self.writer, payload) + + def read_message(self): + if self.mode == "text": + line = self.reader.readline() + else: + data = self.reader.readline() + line = data.decode("utf-8") if data else "" + if not line: + raise EOFError("unexpected EOF while reading line") + return json.loads(line) + + def close(self): + streams = self.closeables if self.closeables else (self.reader, self.writer) + for stream in streams: + if hasattr(stream, "close"): + try: + stream.close() + except OSError: + pass + + +class FDLineReader: + def __init__(self, fd): + self.fd = fd + self.pending = bytearray() + + def readline(self): + while True: + newline = self.pending.find(b"\n") + if newline >= 0: + line = bytes(self.pending[: newline + 1]) + del self.pending[: newline + 1] + return line + chunk = os.read(self.fd, 4096) + if not chunk: + if not self.pending: + return b"" + line = bytes(self.pending) + self.pending.clear() + return line + self.pending.extend(chunk) + + +class State: + def __init__(self, roundtrips, client_io, worker_io, workload, payload_bytes, batch_size): + self.roundtrips = roundtrips + self.client_io = client_io + self.worker_io = worker_io + self.workload = workload + self.payload_bytes = payload_bytes + self.batch_size = batch_size + self.next_request_id = 1 + self.process = None + self.endpoint = None + + +def write_all(fd, data): + view = memoryview(data) + while view: + written = os.write(fd, view) + view = view[written:] + + +def create_text_endpoint(read_raw, write_raw): + reader_buffer = io.BufferedReader(read_raw, buffer_size=8192) + writer_buffer = io.BufferedWriter(write_raw, buffer_size=8192) + reader = io.TextIOWrapper(reader_buffer, encoding="utf-8", newline=None) + writer = io.TextIOWrapper(writer_buffer, encoding="utf-8", newline="\n", line_buffering=False, write_through=False) + return Endpoint("text", reader, writer) + + +def create_buffer_endpoint(read_raw, write_raw): + reader = io.BufferedReader(read_raw, buffer_size=8192) + writer = io.BufferedWriter(write_raw, buffer_size=8192) + return Endpoint("buffer", reader, writer) + + +def create_fd_endpoint(read_fd, write_fd, closeables=()): + return Endpoint("fd", FDLineReader(read_fd), write_fd, closeables) + + +def create_parent_endpoint(process, mode): + if mode == "text": + return create_text_endpoint(process.stdout, process.stdin) + if mode == "buffer": + return create_buffer_endpoint(process.stdout, process.stdin) + return create_fd_endpoint(process.stdout.fileno(), process.stdin.fileno(), (process.stdout, process.stdin)) + + +def create_worker_endpoint(mode): + if mode == "text": + return Endpoint("text", sys.stdin, sys.stdout) + if mode == "buffer": + return Endpoint("buffer", sys.stdin.buffer, sys.stdout.buffer) + return create_fd_endpoint(0, 1) + + +def normalize_email(value): + return EMAIL_RE.sub("", value.strip().lower()) + + +def normalize_phone(value): + digits = NON_DIGIT_RE.sub("", value) + if digits.startswith("00"): + digits = digits[2:] + return digits + + +def mask_email(value): + name, _, domain = value.partition("@") + if not domain: + return "***" + return "%s***@%s" % (name[:1], domain) + + +def mask_phone(value): + if len(value) <= 4: + return "*" * len(value) + return "*" * (len(value) - 4) + value[-4:] + + +def mask_row(row): + email = normalize_email(str(row.get("email", ""))) + phone = normalize_phone(str(row.get("phone", ""))) + return { + "email_normalized": email, + "phone_normalized": phone, + "email_masked": mask_email(email) if email else None, + "phone_masked": mask_phone(phone) if phone else None, + "region": str(row.get("region", "")).upper(), + "source": str(row.get("source", "")).lower(), + } + + +def make_echo_payload(payload_bytes): + if payload_bytes <= 0: + return "" + unit = "payload-" + return (unit * ((payload_bytes // len(unit)) + 1))[:payload_bytes] + + +def make_mask_row(index, payload_bytes): + suffix = make_echo_payload(max(payload_bytes, 8)) + return { + "email": " User%s.%s@Example.COM " % (index, suffix), + "phone": "+49 (170) %04d-%s" % (index, suffix[:8]), + "region": "eu", + "source": "microbench", + } + + +def build_request(kind, request_id, payload_bytes, batch_size): + if kind == "health": + method = "health" + params = {} + elif kind == "echo": + method = "echo" + params = {"payload": make_echo_payload(payload_bytes)} + elif kind == "mask": + method = "mask" + params = make_mask_row(request_id, payload_bytes) + elif kind == "mask_batch": + method = "mask_batch" + params = {"rows": [make_mask_row(request_id + i, payload_bytes) for i in range(batch_size)]} + else: + raise AssertionError("unsupported request kind: %s" % kind) + return {"jsonrpc": "2.0", "id": request_id, "method": method, "params": params} + + +def handle_request(message): + request_id = message.get("id") + method = message.get("method") + params = message.get("params", {}) + if method == "health": + result = {"ok": True, "worker": "jsonrpc-pipe", "protocol": "json-rpc-2.0-ndjson"} + elif method == "echo": + payload = str(dict(params).get("payload", "")) + result = {"ok": True, "echo": payload, "size": len(payload)} + elif method == "mask": + result = {"ok": True, "normalized": mask_row(dict(params))} + elif method == "mask_batch": + rows = [mask_row(dict(row)) for row in dict(params).get("rows", [])] + result = {"ok": True, "normalized": rows, "count": len(rows)} + else: + return {"jsonrpc": "2.0", "id": request_id, "error": {"code": -32601, "message": "method not found"}} + return {"jsonrpc": "2.0", "id": request_id, "result": result} + + +def validate_response(request, response, kind, payload_bytes, batch_size): + if response.get("id") != request["id"]: + raise AssertionError("mismatched response id") + if "error" in response: + raise AssertionError("worker returned error: %s" % (response["error"],)) + result = response.get("result") + if not isinstance(result, dict) or not result.get("ok"): + raise AssertionError("unexpected response payload: %s" % (response,)) + if kind == "echo": + if result.get("echo") != make_echo_payload(payload_bytes): + raise AssertionError("echo payload mismatch") + elif kind == "mask": + expected = mask_row(make_mask_row(int(request["id"]), payload_bytes)) + if result.get("normalized") != expected: + raise AssertionError("mask result mismatch") + elif kind == "mask_batch": + expected_rows = [mask_row(make_mask_row(int(request["id"]) + i, payload_bytes)) for i in range(batch_size)] + if result.get("count") != batch_size or result.get("normalized") != expected_rows: + raise AssertionError("mask_batch result mismatch") + + +def run_roundtrips(state): + completed = 0 + for _ in range(state.roundtrips): + request = build_request(state.workload, state.next_request_id, state.payload_bytes, state.batch_size) + state.next_request_id += 1 + state.endpoint.write_message(request) + response = state.endpoint.read_message() + validate_response(request, response, state.workload, state.payload_bytes, state.batch_size) + completed += 1 + return completed + + +def parse_int(value): + if isinstance(value, int): + return value + return int(str(value).replace("_", "")) + + +def get_subprocess_launcher_args(): + orig_argv = getattr(sys, "orig_argv", None) + if not orig_argv: + return [sys.executable] + launcher_args = [sys.executable] + for arg in orig_argv[1:]: + if not arg.startswith("-"): + break + launcher_args.append(arg) + return launcher_args + + +def __process_args__(roundtrips=500, client_io="text", worker_io="text", workload="mask", payload_bytes=64, batch_size=8): + return [ + parse_int(roundtrips), + str(client_io), + str(worker_io), + str(workload), + parse_int(payload_bytes), + parse_int(batch_size), + ] + + +def __setup__(roundtrips=500, client_io="text", worker_io="text", workload="mask", payload_bytes=64, batch_size=8): + global _STATE + __teardown__() + state = State(roundtrips, client_io, worker_io, workload, payload_bytes, batch_size) + command = [ + *get_subprocess_launcher_args(), + __file__, + "--worker", + "--worker-io=%s" % worker_io, + ] + process = subprocess.Popen( + command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + bufsize=0, + ) + state.process = process + state.endpoint = create_parent_endpoint(process, client_io) + _STATE = state + + +def __benchmark__(roundtrips=500, client_io="text", worker_io="text", workload="mask", payload_bytes=64, batch_size=8): + if _STATE is None: + __setup__(roundtrips, client_io, worker_io, workload, payload_bytes, batch_size) + return run_roundtrips(_STATE) + + +def __teardown__(): + global _STATE + state = _STATE + _STATE = None + if state is None: + return + try: + if state.endpoint is not None: + state.endpoint.close() + finally: + if state.process is not None: + stderr = b"" + try: + stderr = state.process.stderr.read() if state.process.stderr is not None else b"" + except OSError: + pass + return_code = state.process.wait() + if return_code != 0: + raise RuntimeError("worker exited with status %d: %s" % (return_code, stderr.decode("utf-8", errors="replace"))) + + +def run_worker(worker_io): + endpoint = create_worker_endpoint(worker_io) + try: + while True: + try: + request = endpoint.read_message() + except EOFError: + return 0 + endpoint.write_message(handle_request(request)) + finally: + endpoint.close() + + +def run_direct(roundtrips, client_io, worker_io, workload, payload_bytes, batch_size): + start = time.perf_counter() + __setup__(roundtrips, client_io, worker_io, workload, payload_bytes, batch_size) + try: + completed = __benchmark__(roundtrips, client_io, worker_io, workload, payload_bytes, batch_size) + finally: + __teardown__() + wall = time.perf_counter() - start + print("roundtrips=%d" % completed) + print("wall_s=%s" % wall) + print("throughput_ops_s=%s" % (completed / wall if wall else 0.0)) + return 0 + + +def main(argv=None): + parser = argparse.ArgumentParser(description="Strict JSON-RPC-like pipe roundtrip microbenchmark.") + parser.add_argument("--worker", action="store_true") + parser.add_argument("--worker-io", choices=("text", "buffer", "fd"), default="text") + parser.add_argument("--roundtrips", type=parse_int, default=500) + parser.add_argument("--client-io", choices=("text", "buffer", "fd"), default="text") + parser.add_argument("--workload", choices=("health", "echo", "mask", "mask_batch"), default="mask") + parser.add_argument("--payload-bytes", type=parse_int, default=64) + parser.add_argument("--batch-size", type=parse_int, default=8) + args = parser.parse_args(argv) + if args.worker: + return run_worker(args.worker_io) + return run_direct(args.roundtrips, args.client_io, args.worker_io, args.workload, args.payload_bytes, args.batch_size) + + +def run(): + __setup__() + try: + __benchmark__() + finally: + __teardown__() + + +def warmupIterations(): + return 5 + + +def iterations(): + return 10 + + +def summary(): + return { + "name": "OutlierRemovalAverageSummary", + "lower-threshold": 0, + "upper-threshold": 0.3, + } + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/io/BufferedIONodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/io/BufferedIONodes.java index c126f5f927..f6de4820a8 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/io/BufferedIONodes.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/io/BufferedIONodes.java @@ -131,12 +131,12 @@ static boolean isClosed(PBuffered self) { } @SuppressWarnings("unused") - @Specialization(guards = {"self.getBuffer() != null", "self.isFastClosedChecks()"}) + @Specialization(guards = {"self.getBuffer() != null", "self.hasFileIORaw()"}) static boolean isClosedFileIO(PBuffered self) { return self.getFileIORaw().isClosed(); } - @Specialization(guards = {"self.getBuffer() != null", "!self.isFastClosedChecks()"}) + @Specialization(guards = {"self.getBuffer() != null", "!self.hasFileIORaw()"}) static boolean isClosedBuffered(VirtualFrame frame, Node inliningTarget, PBuffered self, @Cached PyObjectGetAttr getAttr, @Cached PyObjectIsTrueNode isTrue) { diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/io/BufferedReaderMixinBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/io/BufferedReaderMixinBuiltins.java index 8f5fb49222..f32954f9e4 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/io/BufferedReaderMixinBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/io/BufferedReaderMixinBuiltins.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/io/BufferedWriterNodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/io/BufferedWriterNodes.java index 5c7a52b178..76944609fa 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/io/BufferedWriterNodes.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/io/BufferedWriterNodes.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -48,11 +48,16 @@ import static com.oracle.graal.python.builtins.modules.io.IONodes.T_WRITE; import static com.oracle.graal.python.nodes.ErrorMessages.IO_S_INVALID_LENGTH; import static com.oracle.graal.python.nodes.ErrorMessages.WRITE_COULD_NOT_COMPLETE_WITHOUT_BLOCKING; +import static com.oracle.graal.python.nodes.ErrorMessages.IO_CLOSED; +import static com.oracle.graal.python.nodes.ErrorMessages.FILE_NOT_OPEN_FOR_S; +import static com.oracle.graal.python.runtime.exception.PythonErrorType.IOUnsupportedOperation; import static com.oracle.graal.python.runtime.exception.PythonErrorType.OSError; import static com.oracle.graal.python.runtime.exception.PythonErrorType.ValueError; import com.oracle.graal.python.PythonLanguage; +import com.oracle.graal.python.builtins.modules.PosixModuleBuiltins; import com.oracle.graal.python.builtins.objects.PNone; +import com.oracle.graal.python.builtins.objects.exception.OSErrorEnum; import com.oracle.graal.python.builtins.objects.buffer.PythonBufferAccessLibrary; import com.oracle.graal.python.builtins.objects.bytes.PBytes; import com.oracle.graal.python.builtins.objects.common.SequenceStorageNodes; @@ -61,8 +66,12 @@ import com.oracle.graal.python.lib.PyNumberAsSizeNode; import com.oracle.graal.python.lib.PyObjectCallMethodObjArgs; import com.oracle.graal.python.nodes.PNodeWithContext; +import com.oracle.graal.python.nodes.PConstructAndRaiseNode; import com.oracle.graal.python.nodes.PRaiseNode; import com.oracle.graal.python.nodes.object.BuiltinClassProfiles.IsBuiltinObjectProfile; +import com.oracle.graal.python.runtime.PosixSupportLibrary; +import com.oracle.graal.python.runtime.PosixSupportLibrary.PosixException; +import com.oracle.graal.python.runtime.PythonContext; import com.oracle.graal.python.runtime.exception.PException; import com.oracle.graal.python.runtime.object.PFactory; import com.oracle.graal.python.util.PythonUtils; @@ -72,8 +81,10 @@ import com.oracle.truffle.api.dsl.GenerateInline; import com.oracle.truffle.api.dsl.Specialization; import com.oracle.truffle.api.frame.VirtualFrame; +import com.oracle.truffle.api.profiles.InlinedBranchProfile; import com.oracle.truffle.api.library.CachedLibrary; import com.oracle.truffle.api.nodes.Node; +import com.oracle.graal.python.runtime.GilNode; public class BufferedWriterNodes { @@ -222,7 +233,40 @@ abstract static class RawWriteNode extends PNodeWithContext { /** * implementation of cpython/Modules/_io/bufferedio.c:_bufferedwriter_raw_write */ - @Specialization + @SuppressWarnings("truffle-sharing") + @Specialization(guards = "self.hasFileIORaw()") + static int bufferedwriterRawWriteFileIO(VirtualFrame frame, Node inliningTarget, PBuffered self, byte[] buf, int len, + @Bind PythonContext context, + @CachedLibrary("context.getPosixSupport()") PosixSupportLibrary posixLib, + @Cached InlinedBranchProfile errorProfile, + @Cached GilNode gil, + @Cached PConstructAndRaiseNode.Lazy constructAndRaiseNode, + @Cached PRaiseNode raiseNode) { + PFileIO fileIO = self.getFileIORaw(); + if (fileIO.isClosed()) { + throw raiseNode.raise(inliningTarget, ValueError, IO_CLOSED); + } + if (!fileIO.isWritable()) { + throw raiseNode.raise(inliningTarget, IOUnsupportedOperation, FILE_NOT_OPEN_FOR_S, "writing"); + } + final int n; + try { + n = Math.toIntExact(PosixModuleBuiltins.WriteNode.write(fileIO.getFD(), buf, len, + inliningTarget, posixLib, context.getPosixSupport(), errorProfile, gil)); + } catch (PosixException e) { + if (e.getErrorCode() == OSErrorEnum.EAGAIN.getNumber()) { + return -2; + } + throw constructAndRaiseNode.get(inliningTarget).raiseOSErrorFromPosixException(frame, e); + } + if (n > 0 && self.getAbsPos() != -1) { + self.incAbsPos(n); + } + return n; + } + + @SuppressWarnings("truffle-sharing") + @Specialization(guards = "!self.hasFileIORaw()") static int bufferedwriterRawWrite(VirtualFrame frame, Node inliningTarget, PBuffered self, byte[] buf, int len, @Bind PythonLanguage language, @Cached PyObjectCallMethodObjArgs callMethod, @@ -272,8 +316,16 @@ protected static void bufferedwriterFlushUnlocked(VirtualFrame frame, PBuffered self.incRawPos(-rewind); } while (self.getWritePos() < self.getWriteEnd()) { - byte[] buf = PythonUtils.arrayCopyOfRange(self.getBuffer(), self.getWritePos(), self.getWriteEnd()); - int n = rawWriteNode.execute(frame, inliningTarget, self, buf, buf.length); + byte[] buf; + int len; + if (self.hasFileIORaw() && self.getWritePos() == 0) { + buf = self.getBuffer(); + len = self.getWriteEnd(); + } else { + buf = PythonUtils.arrayCopyOfRange(self.getBuffer(), self.getWritePos(), self.getWriteEnd()); + len = buf.length; + } + int n = rawWriteNode.execute(frame, inliningTarget, self, buf, len); if (n == -2) { throw raiseBlockingIOError.get(inliningTarget).raiseEAGAIN(WRITE_COULD_NOT_COMPLETE_WITHOUT_BLOCKING, 0); } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/io/PBuffered.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/io/PBuffered.java index bbcf567d5c..bf71cb80ba 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/io/PBuffered.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/io/PBuffered.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -158,7 +158,7 @@ public void setFinalizing(boolean finalizing) { this.finalizing = finalizing; } - public boolean isFastClosedChecks() { + public boolean hasFileIORaw() { return fileioRaw != null; } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/io/PTextIO.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/io/PTextIO.java index 200ff15b56..155ac163ad 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/io/PTextIO.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/io/PTextIO.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -41,13 +41,9 @@ package com.oracle.graal.python.builtins.modules.io; import static com.oracle.graal.python.builtins.objects.bytes.BytesUtils.append; -import static com.oracle.graal.python.builtins.objects.bytes.BytesUtils.createOutputStream; -import static com.oracle.graal.python.builtins.objects.bytes.BytesUtils.toByteArray; import static com.oracle.graal.python.nodes.StringLiterals.T_EMPTY_STRING; import static com.oracle.graal.python.util.PythonUtils.TS_ENCODING; -import java.io.ByteArrayOutputStream; - import com.oracle.graal.python.builtins.objects.ints.IntBuiltins; import com.oracle.graal.python.builtins.objects.ints.IntNodes; import com.oracle.graal.python.builtins.objects.ints.PInt; @@ -93,7 +89,7 @@ public final class PTextIO extends PTextIOBase { private int decodedCharsUsed; /* offset (in code points) into _decoded_chars for read() */ private int decodedCharsLen; /* code point length of decodedChars */ - private ByteArrayOutputStream pendingBytes; // data waiting to be written. + private PendingBytesOutputStream pendingBytes; // data waiting to be written. /* * snapshot is either NULL, or a tuple (dec_flags, next_input) where dec_flags is the second @@ -112,7 +108,7 @@ public final class PTextIO extends PTextIOBase { public PTextIO(Object cls, Shape instanceShape) { super(cls, instanceShape); - pendingBytes = createOutputStream(); + pendingBytes = new PendingBytesOutputStream(); } @Override @@ -324,11 +320,11 @@ TruffleString consumeAllDecodedChars(TruffleString.SubstringNode substringNode, } public void clearPendingBytes() { - pendingBytes = createOutputStream(); + pendingBytes = new PendingBytesOutputStream(); } - public byte[] getAndClearPendingBytes() { - byte[] b = toByteArray(pendingBytes); + public PendingBytesOutputStream getAndClearPendingBytes() { + PendingBytesOutputStream b = pendingBytes; clearPendingBytes(); return b; } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/io/PendingBytesOutputStream.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/io/PendingBytesOutputStream.java new file mode 100644 index 0000000000..809711b9f3 --- /dev/null +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/io/PendingBytesOutputStream.java @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The Universal Permissive License (UPL), Version 1.0 + * + * Subject to the condition set forth below, permission is hereby granted to any + * person obtaining a copy of this software, associated documentation and/or + * data (collectively the "Software"), free of charge and under any and all + * copyright rights in the Software, and any and all patent rights owned or + * freely licensable by each licensor hereunder covering either (i) the + * unmodified Software as contributed to or provided by such licensor, or (ii) + * the Larger Works (as defined below), to deal in both + * + * (a) the Software, and + * + * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if + * one is included with the Software each a "Larger Work" to which the Software + * is contributed by such licensors), + * + * without restriction, including without limitation the rights to copy, create + * derivative works of, display, perform, and distribute the Software and make, + * use, sell, offer for sale, import, export, have made, and have sold the + * Software and the Larger Work(s), and to sublicense the foregoing rights on + * either these or other terms. + * + * This license is subject to the following condition: + * + * The above copyright notice and either this complete permission notice or at a + * minimum a reference to the UPL must be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package com.oracle.graal.python.builtins.modules.io; + +import java.io.ByteArrayOutputStream; + +final class PendingBytesOutputStream extends ByteArrayOutputStream { + + PendingBytesOutputStream() { + super(); + } + + PendingBytesOutputStream(int size) { + super(size); + } + + byte[] getBuffer() { + return buf; + } + + int getCount() { + return count; + } +} diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/io/TextIOWrapperNodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/io/TextIOWrapperNodes.java index 5eb40a3447..71d9832ae4 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/io/TextIOWrapperNodes.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/io/TextIOWrapperNodes.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -207,8 +207,8 @@ static void nothingTodo(@SuppressWarnings("unused") PTextIO self) { static void writeflush(VirtualFrame frame, Node inliningTarget, PTextIO self, @Bind PythonLanguage language, @Cached PyObjectCallMethodObjArgs callMethod) { - byte[] pending = self.getAndClearPendingBytes(); - PBytes b = PFactory.createBytes(language, pending); + PendingBytesOutputStream pending = self.getAndClearPendingBytes(); + PBytes b = PFactory.createBytes(language, pending.getBuffer(), pending.getCount()); callMethod.execute(frame, inliningTarget, self.getBuffer(), T_WRITE, b); // TODO: check _PyIO_trap_eintr } @@ -884,7 +884,7 @@ static void init(VirtualFrame frame, Node inliningTarget, PTextIO self, Object b if (buffer instanceof PBuffered) { /* Cache the raw FileIO object to speed up 'closed' checks */ - if (((PBuffered) buffer).isFastClosedChecks()) { + if (((PBuffered) buffer).hasFileIORaw()) { PFileIO f = ((PBuffered) buffer).getFileIORaw(); self.setFileIO(f); } diff --git a/mx.graalpython/mx_graalpython_bench_param.py b/mx.graalpython/mx_graalpython_bench_param.py index 78d5bf06c6..fe9eb3ef0e 100644 --- a/mx.graalpython/mx_graalpython_bench_param.py +++ b/mx.graalpython/mx_graalpython_bench_param.py @@ -121,6 +121,7 @@ 'virtualize-in-try-catch-oom': ITER_10, 'phase_shift_warmup_baseline': ITER_5 + ['--self-measurement'] + ['500'], 'phase_shift_warmup': ITER_3 + ['--self-measurement'] + ['1600', '500'], + 'jsonrpc-pipe': ITER_10 + ['500', 'text', 'text', 'mask', '64'], 'startup': ITER_5 + ['50'], 'startup-imports': ITER_5 + ['20'], } @@ -130,6 +131,7 @@ 'nano-arith': ITER_6 + WARMUP_2, 'nano-loop': ITER_6 + WARMUP_2, 'nano-if': ITER_6 + WARMUP_2, + 'jsonrpc-pipe': ITER_6 + WARMUP_2 + ['100', 'text', 'text', 'mask', '64'], 'arith-modulo-sized': ITER_6 + WARMUP_2 + ['1'], 'if-generic': ITER_10 + WARMUP_2 + ['500000'], 'if-generic-non-builtin': ITER_10 + WARMUP_2 + ['500000'], diff --git a/scripts/profile-jsonrpc-pipe-async-buffer.sh b/scripts/profile-jsonrpc-pipe-async-buffer.sh new file mode 100755 index 0000000000..5ca134c571 --- /dev/null +++ b/scripts/profile-jsonrpc-pipe-async-buffer.sh @@ -0,0 +1,51 @@ +# Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +OUT="${1:-/tmp/jsonrpc-ee-worker-buffer.svg}" +REQS="${REQS:-30000}" +GRAALPY="${GRAALPY:-$ROOT/mxbuild/linux-amd64/GRAALPY_JVM_STANDALONE/bin/graalpy}" +exec python3 "$ROOT/scripts/profile_jsonrpc_pipe_worker.py" \ + --graalpy "$GRAALPY" \ + --worker-io buffer \ + --profiler async \ + --requests "$REQS" \ + --output "$OUT" diff --git a/scripts/profile-jsonrpc-pipe-async-text.sh b/scripts/profile-jsonrpc-pipe-async-text.sh new file mode 100755 index 0000000000..2b75b1741d --- /dev/null +++ b/scripts/profile-jsonrpc-pipe-async-text.sh @@ -0,0 +1,51 @@ +# Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +OUT="${1:-/tmp/jsonrpc-ee-worker-text.svg}" +REQS="${REQS:-30000}" +GRAALPY="${GRAALPY:-$ROOT/mxbuild/linux-amd64/GRAALPY_JVM_STANDALONE/bin/graalpy}" +exec python3 "$ROOT/scripts/profile_jsonrpc_pipe_worker.py" \ + --graalpy "$GRAALPY" \ + --worker-io text \ + --profiler async \ + --requests "$REQS" \ + --output "$OUT" diff --git a/scripts/profile-jsonrpc-pipe-gprofng-buffer.sh b/scripts/profile-jsonrpc-pipe-gprofng-buffer.sh new file mode 100755 index 0000000000..3aed4f4026 --- /dev/null +++ b/scripts/profile-jsonrpc-pipe-gprofng-buffer.sh @@ -0,0 +1,51 @@ +# Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +OUT="${1:-/tmp/jsonrpc-ee-worker-buffer.er}" +REQS="${REQS:-30000}" +GRAALPY="${GRAALPY:-$ROOT/mxbuild/linux-amd64/GRAALPY_JVM_STANDALONE/bin/graalpy}" +exec python3 "$ROOT/scripts/profile_jsonrpc_pipe_worker.py" \ + --graalpy "$GRAALPY" \ + --worker-io buffer \ + --profiler gprofng \ + --requests "$REQS" \ + --output "$OUT" diff --git a/scripts/profile-jsonrpc-pipe-gprofng-text.sh b/scripts/profile-jsonrpc-pipe-gprofng-text.sh new file mode 100755 index 0000000000..63be3fb887 --- /dev/null +++ b/scripts/profile-jsonrpc-pipe-gprofng-text.sh @@ -0,0 +1,51 @@ +# Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +OUT="${1:-/tmp/jsonrpc-ee-worker-text.er}" +REQS="${REQS:-30000}" +GRAALPY="${GRAALPY:-$ROOT/mxbuild/linux-amd64/GRAALPY_JVM_STANDALONE/bin/graalpy}" +exec python3 "$ROOT/scripts/profile_jsonrpc_pipe_worker.py" \ + --graalpy "$GRAALPY" \ + --worker-io text \ + --profiler gprofng \ + --requests "$REQS" \ + --output "$OUT" diff --git a/scripts/profile_jsonrpc_pipe_worker.py b/scripts/profile_jsonrpc_pipe_worker.py new file mode 100755 index 0000000000..aed3bb4b90 --- /dev/null +++ b/scripts/profile_jsonrpc_pipe_worker.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import subprocess +import sys +from typing import BinaryIO, TextIO + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Drive the jsonrpc-pipe worker under a profiler.") + parser.add_argument("--graalpy", required=True, help="Path to GraalPy launcher") + parser.add_argument("--worker-io", choices=("text", "buffer"), required=True) + parser.add_argument("--profiler", choices=("async", "gprofng"), required=True) + parser.add_argument("--requests", type=int, default=30000) + parser.add_argument("--output", required=True, help="Profile output file (async) or experiment dir (gprofng)") + parser.add_argument("--async-profiler-dir", default="/tmp/async-profiler-1.8.3-linux-x64") + parser.add_argument("--benchmark", default="graalpython/com.oracle.graal.python.benchmarks/python/micro/jsonrpc-pipe.py") + return parser.parse_args() + + +def build_worker_cmd(args: argparse.Namespace) -> list[str]: + benchmark = str(Path(args.benchmark).resolve()) + worker = [args.graalpy, benchmark, "--worker", f"--worker-io={args.worker_io}"] + if args.profiler == "async": + lib = Path(args.async_profiler_dir) / "build" / "libasyncProfiler.so" + return [ + args.graalpy, + f"--vm.agentpath:{lib}=start,event=cpu,file={args.output}", + "--vm.XX:+UnlockDiagnosticVMOptions", + "--vm.XX:+DebugNonSafepoints", + benchmark, + "--worker", + f"--worker-io={args.worker_io}", + ] + return [ + "gprofng", + "collect", + "app", + "-O", + args.output, + "-F", + "off", + "--", + *worker, + ] + + +def make_request(index: int) -> dict[str, object]: + return { + "jsonrpc": "2.0", + "id": index, + "method": "mask", + "params": { + "email": f" User{index}.payload-payload@Example.COM ", + "phone": f"+49 (170) {index:04d}-payload-", + "region": "eu", + "source": "microbench", + }, + } + + +def read_json_line_text(stream: TextIO, stderr: TextIO) -> dict[str, object]: + while True: + line = stream.readline() + if not line: + raise RuntimeError(f"worker terminated early: {stderr.read()}") + if line.lstrip().startswith("{"): + return json.loads(line) + + +def read_json_line_binary(stream: BinaryIO, stderr: BinaryIO) -> dict[str, object]: + while True: + line = stream.readline() + if not line: + raise RuntimeError(f"worker terminated early: {stderr.read().decode('utf-8', errors='replace')}") + if line.lstrip().startswith(b"{"): + return json.loads(line) + + +def drive_text(process: subprocess.Popen[str], requests: int) -> None: + assert process.stdin is not None + assert process.stdout is not None + assert process.stderr is not None + for i in range(requests): + process.stdin.write(json.dumps(make_request(i), separators=(",", ":")) + "\n") + process.stdin.flush() + read_json_line_text(process.stdout, process.stderr) + + +def drive_binary(process: subprocess.Popen[bytes], requests: int) -> None: + assert process.stdin is not None + assert process.stdout is not None + assert process.stderr is not None + for i in range(requests): + payload = (json.dumps(make_request(i), separators=(",", ":")) + "\n").encode("utf-8") + process.stdin.write(payload) + process.stdin.flush() + read_json_line_binary(process.stdout, process.stderr) + + +def main() -> int: + args = parse_args() + output = Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + if args.profiler == "gprofng" and output.exists(): + if output.is_dir(): + subprocess.check_call(["rm", "-rf", str(output)]) + else: + output.unlink() + cmd = build_worker_cmd(args) + process_cwd = None + if args.profiler == "gprofng": + process_cwd = str(output.parent) + cmd[4] = output.name + text_mode = args.worker_io == "text" + if text_mode: + process = subprocess.Popen( + cmd, + cwd=process_cwd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + encoding="utf-8", + bufsize=1, + ) + try: + drive_text(process, args.requests) + process.stdin.close() + rc = process.wait(timeout=120) + if rc != 0: + raise RuntimeError(process.stderr.read()) + sys.stdout.write(process.stderr.read()) + finally: + if process.poll() is None: + process.kill() + else: + process = subprocess.Popen( + cmd, + cwd=process_cwd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + bufsize=0, + ) + try: + drive_binary(process, args.requests) + process.stdin.close() + rc = process.wait(timeout=120) + if rc != 0: + raise RuntimeError(process.stderr.read().decode("utf-8", errors="replace")) + sys.stdout.write(process.stderr.read().decode("utf-8", errors="replace")) + finally: + if process.poll() is None: + process.kill() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())