Add a Python package for fetch_artifact.

We keep duplicating this function all over the place. Factor it out
and modernize it.

Test: all the test commands named in the readme
Bug: None
Change-Id: I8d90a9c101b9cf83d62d02141f40c6c3666bdf94

python-packages/fetchartifact/OWNERS

@@ -0,0 +1 @@
+danalbert@google.com

python-packages/fetchartifact/README.md

@@ -0,0 +1,64 @@
+# fetchartifact
+
+This is a Python interface to http://go/fetchartifact, which is used for
+fetching artifacts from http://go/ab.
+
+## Usage
+
+```python
+from fetchartifact import fetchartifact
+
+
+async def main() -> None:
+    artifacts = await fetch_artifact(
+        branch="aosp-master-ndk",
+        target="linux",
+        build="1234",
+        pattern="android-ndk-*.zip",
+    )
+    for artifact in artifacts:
+        print(f"Downloaded {artifact}")
+```
+
+## Development
+
+For first time set-up, install https://python-poetry.org/, then run
+`poetry install` to install the project's dependencies.
+
+This project uses mypy and pylint for linting, black and isort for
+auto-formatting, and pytest for testing. All of these tools will be installed
+automatically, but you may want to configure editor integration for them.
+
+To run any of the tools poetry installed, you can either prefix all your
+commands with `poetry run` (as in `poetry run pytest`), or you can run
+`poetry shell` to enter a shell with all the tools on the `PATH`. The following
+instructions assume you've run `poetry shell` first.
+
+To run the linters:
+
+```bash
+mypy fetchartifact tests
+pylint fetchartifact tests
+```
+
+To auto-format the code (though I recommend configuring your editor to do this
+on save):
+
+```bash
+isort .
+black .
+```
+
+To run the tests and generate coverage:
+
+```bash
+pytest --cov=fetchartifact
+```
+
+Optionally, pass `--cov-report=html` to generate an HTML report, or
+`--cov-report=xml` to generate an XML report for your editor.
+
+Some tests require network access. If you need to run the tests in an
+environment that cannot access the Android build servers, add
+`-m "not requires_network"` to skip those tests. Only a mock service can be
+tested without network access.

python-packages/fetchartifact/fetchartifact/__init__.py

@@ -0,0 +1,112 @@
+#
+# Copyright (C) 2023 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""A Python interface to https://android.googlesource.com/tools/fetch_artifact/."""
+import logging
+import urllib
+from collections.abc import AsyncIterable
+from logging import Logger
+from typing import cast
+
+from aiohttp import ClientSession
+
+_DEFAULT_QUERY_URL_BASE = "https://androidbuildinternal.googleapis.com"
+
+
+def _logger() -> Logger:
+    return logging.getLogger("fetchartifact")
+
+
+def _make_download_url(
+    target: str,
+    build_id: str,
+    artifact_name: str,
+    query_url_base: str,
+) -> str:
+    """Constructs the download URL.
+
+    Args:
+        target: Name of the build target from which to fetch the artifact.
+        build_id: ID of the build from which to fetch the artifact.
+        artifact_name: Name of the artifact to fetch.
+
+    Returns:
+        URL for the given artifact.
+    """
+    # The Android build API does not handle / in artifact names, but urllib.parse.quote
+    # thinks those are safe by default. We need to escape them.
+    artifact_name = urllib.parse.quote(artifact_name, safe="")
+    return (
+        f"{query_url_base}/android/internal/build/v3/builds/{build_id}/{target}/"
+        f"attempts/latest/artifacts/{artifact_name}/url"
+    )
+
+
+async def fetch_artifact(
+    target: str,
+    build_id: str,
+    artifact_name: str,
+    session: ClientSession,
+    query_url_base: str = _DEFAULT_QUERY_URL_BASE,
+) -> bytes:
+    """Fetches an artifact from the build server.
+
+    Args:
+        target: Name of the build target from which to fetch the artifact.
+        build_id: ID of the build from which to fetch the artifact.
+        artifact_name: Name of the artifact to fetch.
+        session: The aiohttp ClientSession to use. If omitted, one will be created and
+            destroyed for every call.
+        query_url_base: The base of the endpoint used for querying download URLs. Uses
+            the android build service by default, but can be replaced for testing.
+
+    Returns:
+        The bytes of the downloaded artifact.
+    """
+    download_url = _make_download_url(target, build_id, artifact_name, query_url_base)
+    _logger().debug("Beginning download from %s", download_url)
+    async with session.get(download_url) as response:
+        response.raise_for_status()
+        return await response.read()
+
+
+async def fetch_artifact_chunked(
+    target: str,
+    build_id: str,
+    artifact_name: str,
+    session: ClientSession,
+    chunk_size: int = 16 * 1024 * 1024,
+    query_url_base: str = _DEFAULT_QUERY_URL_BASE,
+) -> AsyncIterable[bytes]:
+    """Fetches an artifact from the build server.
+
+    Args:
+        target: Name of the build target from which to fetch the artifact.
+        build_id: ID of the build from which to fetch the artifact.
+        artifact_name: Name of the artifact to fetch.
+        session: The aiohttp ClientSession to use. If omitted, one will be created and
+            destroyed for every call.
+        query_url_base: The base of the endpoint used for querying download URLs. Uses
+            the android build service by default, but can be replaced for testing.
+
+    Returns:
+        Async iterable bytes of the artifact contents.
+    """
+    download_url = _make_download_url(target, build_id, artifact_name, query_url_base)
+    _logger().debug("Beginning download from %s", download_url)
+    async with session.get(download_url) as response:
+        response.raise_for_status()
+        async for chunk in response.content.iter_chunked(chunk_size):
+            yield chunk

python-packages/fetchartifact/pyproject.toml

@@ -0,0 +1,60 @@
+[tool.poetry]
+name = "fetchartifact"
+version = "0.1.0"
+description = "Python library for https://android.googlesource.com/tools/fetch_artifact/."
+authors = ["The Android Open Source Project"]
+license = "Apache-2.0"
+
+[tool.poetry.dependencies]
+python = "^3.10"
+aiohttp = "^3.8.4"
+
+[tool.poetry.group.dev.dependencies]
+mypy = "^1.2.0"
+pylint = "^2.17.2"
+black = "^23.3.0"
+pytest = "^7.2.2"
+pytest-asyncio = "^0.21.0"
+isort = "^5.12.0"
+pytest-aiohttp = "^1.0.4"
+pytest-cov = "^4.0.0"
+
+[tool.coverage.report]
+fail_under = 100
+
+[tool.pytest.ini_options]
+addopts = "--strict-markers"
+asyncio_mode = "auto"
+markers = [
+    "requires_network: marks a test that requires network access"
+]
+xfail_strict = true
+
+[tool.mypy]
+check_untyped_defs = true
+disallow_any_generics = true
+disallow_any_unimported = true
+disallow_subclassing_any = true
+disallow_untyped_decorators = true
+disallow_untyped_defs = true
+follow_imports = "silent"
+implicit_reexport = false
+namespace_packages = true
+no_implicit_optional = true
+show_error_codes = true
+strict_equality = true
+warn_redundant_casts = true
+warn_return_any = true
+warn_unreachable = true
+warn_unused_configs = true
+warn_unused_ignores = true
+
+[tool.pylint."MESSAGES CONTROL"]
+disable = "duplicate-code,too-many-arguments"
+
+[tool.isort]
+profile = "black"
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"

python-packages/fetchartifact/tests/__init__.py

@@ -0,0 +1,15 @@
+#
+# Copyright (C) 2023 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#

python-packages/fetchartifact/tests/test_fetchartifact.py

@@ -0,0 +1,101 @@
+#
+# Copyright (C) 2023 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Tests for fetchartifact."""
+from typing import cast
+
+import pytest
+from aiohttp import ClientResponseError, ClientSession
+from aiohttp.test_utils import TestClient
+from aiohttp.web import Application, Request, Response
+
+from fetchartifact import fetch_artifact, fetch_artifact_chunked
+
+TEST_BUILD_ID = "1234"
+TEST_TARGET = "linux"
+TEST_ARTIFACT_NAME = "output.zip"
+TEST_DOWNLOAD_URL = (
+    f"/android/internal/build/v3/builds/{TEST_BUILD_ID}/{TEST_TARGET}/"
+    f"attempts/latest/artifacts/{TEST_ARTIFACT_NAME}/url"
+)
+TEST_RESPONSE = b"Hello, world!"
+
+
+@pytest.fixture(name="android_ci_client")
+async def fixture_android_ci_client(aiohttp_client: type[TestClient]) -> TestClient:
+    """Fixture for mocking the Android CI APIs."""
+
+    async def download(_request: Request) -> Response:
+        return Response(text=TEST_RESPONSE.decode("utf-8"))
+
+    app = Application()
+    app.router.add_get(TEST_DOWNLOAD_URL, download)
+    return await aiohttp_client(app)  # type: ignore
+
+
+async def test_fetch_artifact(android_ci_client: TestClient) -> None:
+    """Tests that the download URL is queried."""
+    assert TEST_RESPONSE == await fetch_artifact(
+        TEST_TARGET,
+        TEST_BUILD_ID,
+        TEST_ARTIFACT_NAME,
+        cast(ClientSession, android_ci_client),
+        query_url_base="",
+    )
+
+
+async def test_fetch_artifact_chunked(android_ci_client: TestClient) -> None:
+    """Tests that the full file contents are downloaded."""
+    assert [c.encode("utf-8") for c in TEST_RESPONSE.decode("utf-8")] == [
+        chunk
+        async for chunk in fetch_artifact_chunked(
+            TEST_TARGET,
+            TEST_BUILD_ID,
+            TEST_ARTIFACT_NAME,
+            cast(ClientSession, android_ci_client),
+            chunk_size=1,
+            query_url_base="",
+        )
+    ]
+
+
+async def test_failure_raises(android_ci_client: TestClient) -> None:
+    """Tests that fetch failure raises an exception."""
+    with pytest.raises(ClientResponseError):
+        await fetch_artifact(
+            TEST_TARGET,
+            TEST_BUILD_ID,
+            TEST_ARTIFACT_NAME,
+            cast(ClientSession, android_ci_client),
+            query_url_base="/bad",
+        )
+
+    with pytest.raises(ClientResponseError):
+        async for _chunk in fetch_artifact_chunked(
+            TEST_TARGET,
+            TEST_BUILD_ID,
+            TEST_ARTIFACT_NAME,
+            cast(ClientSession, android_ci_client),
+            query_url_base="/bad",
+        ):
+            pass
+
+
+@pytest.mark.requires_network
+async def test_real_artifact() -> None:
+    """Tests with a real artifact. Requires an internet connection."""
+    async with ClientSession() as session:
+        contents = await fetch_artifact("linux", "9945621", "logs/SUCCEEDED", session)
+        assert contents == b"1681499053\n"