diff --git a/.github/workflows/build-manifests.yml b/.github/workflows/build-manifests.yml index ff43763..9a1b43c 100644 --- a/.github/workflows/build-manifests.yml +++ b/.github/workflows/build-manifests.yml @@ -1,10 +1,8 @@ -name: Build and publish releases artifacts +name: Publish redirect page on: push: branches: main - schedule: - - cron: "0 1 * * *" # Allows you to run this workflow manually from the Actions tab workflow_dispatch: @@ -18,7 +16,7 @@ concurrency: jobs: deploy: - name: Build and Deploy + name: Deploy environment: name: github-pages url: ${{ steps.deployment.outputs.page_url }} @@ -33,27 +31,10 @@ jobs: with: persist-credentials: false - - name: Set up Python 3.12 - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: "3.12" - pip-version: "26.2.2" - - - name: Install dependencies - run: | - cd utils - pip install -r requirements.txt - - - name: Build latest.ddb - run: | - cd utils - python3 fetch_releases_from_stac.py - - name: Copy output to publish directory run: | mkdir publish cp 404.html publish/ - cp utils/latest.ddb publish/ - name: Setup Pages uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6.0.0 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 397b402..0000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: Test utils - -on: - push: - branches: main - pull_request: - -permissions: - contents: read - -concurrency: - group: "test-${{ github.ref }}" - cancel-in-progress: true - -jobs: - test: - name: Run unit tests - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Set up Python 3.12 - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: "3.12" - pip-version: "26.2.2" - - - name: Install dependencies - run: | - cd utils - pip install -r requirements-test.txt - - - name: Run tests - run: | - cd utils - python -m pytest tests/ -v diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 41c33c8..0000000 --- a/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ - -__pycache__/ -*.pyc diff --git a/404.html b/404.html index 3929ce1..272a5e6 100644 --- a/404.html +++ b/404.html @@ -86,7 +86,8 @@

This resource has moved

The file you requested is no longer published here. Release discovery is now handled by the Overture STAC catalog - (SpatioTemporal Asset Catalog). For upcoming release dates, see the + (SpatioTemporal Asset Catalog), which also serves + latest.ddb. For upcoming release dates, see the release calendar.

diff --git a/CODEOWNERS b/CODEOWNERS index 78ef049..e881232 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,5 +1,2 @@ # Default owners for everything in the repo * @overturemaps/omf-public-reviewers - -# Devops -/.github @lowlydba @overturemaps/omf-public-reviewers diff --git a/README.md b/README.md index c3497bd..fda629e 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,42 @@ -# Welcome to the Overture Maps Data Repo +

+ + Overture Maps Foundation + +

-We welcome feedback about Overture Maps data in the [Discussions](https://github.com/OvertureMaps/data/discussions/new/choose). Feedback on the data *schema*, is best provided in the [Discussions in the *schema* repository](https://github.com/OvertureMaps/schema/discussions). +

Overture Maps Data

-## Overture Maps Data +

+ Open map data in cloud-native Parquet, free to use under open licenses. +

+ +

+ Docs · + Schema · + STAC · + Release calendar · + Discussions +

+ +A reference for where Overture Maps data lives, how it is laid out, and where to send feedback. For tutorials, guides, and examples, see [docs.overturemaps.org](https://docs.overturemaps.org). + +## Contents - [Location](#location) - - [Amazon S3](#amazon-s3) - - [Microsoft Azure](#microsoft-azure) - [Parquet Schema](#parquet-schema) - [Release Discovery](#release-discovery) -- [Data Release Feedback](#data-release-feedback) +- [Feedback](#feedback) -### Location +## Location -Overture Maps data is available in cloud-native [Parquet](https://parquet.apache.org/docs/) format. -There is no single Overture "entire planet" file to download. Instead, the data is partitioned by both `theme` and `type` and made available on Amazon S3 and Microsoft Azure Blob Storage. Read more about Overture data and schema at [docs.overturemaps.org](https://docs.overturemaps.org). +Overture Maps data is available in cloud-native [Parquet](https://parquet.apache.org/docs/) format. There is no single "entire planet" file to download. The data is partitioned by `theme` and `type` and hosted on Amazon S3 and Microsoft Azure Blob Storage. -The latest data release is available at the following locations: +Replace `` below with a release version from the [STAC](#release-discovery), for example `2026-10-21.0`. -#### Amazon S3 +### Amazon S3 ```text -s3://overturemaps-us-west-2/release/__OVERTURE_RELEASE/ +s3://overturemaps-us-west-2/release// |-- theme=addresses/ |-- theme=base/ |-- theme=buildings/ @@ -30,35 +45,38 @@ s3://overturemaps-us-west-2/release/__OVERTURE_RELEASE/ |-- theme=transportation/ ``` -#### Microsoft Azure +### Microsoft Azure ```text -https://overturemapswestus2.blob.core.windows.net/release/__OVERTURE_RELEASE/ - |- theme=addresses - |- theme=base - |- theme=buildings - |- theme=divisions - |- theme=places - |- theme=transportation +https://overturemapswestus2.blob.core.windows.net/release// + |-- theme=addresses/ + |-- theme=base/ + |-- theme=buildings/ + |-- theme=divisions/ + |-- theme=places/ + |-- theme=transportation/ ``` -### Parquet Schema +## Parquet Schema -The Parquet files match the [Overture Data Schema](https://docs.overturemaps.org/schema/) -for each theme with the following enhancements: +The Parquet files match the [Overture Data Schema](https://docs.overturemaps.org/schema/) for each theme, with these additions: 1. The `id` column contains unique identifiers in the [Global Entity Reference System (GERS)](https://docs.overturemaps.org/gers/) format. -2. The `bbox` column is a `struct` with the following attributes: - `xmin`, `xmax`, `ymin`, `ymax`. This column allows you to craft more - efficient spatial queries when running SQL against the cloud. -3. The `geometry` column is encoded as WKB (the files are geoparquet). +2. The `bbox` column is a `struct` with `xmin`, `xmax`, `ymin`, and `ymax`. Filter on it to make spatial queries against cloud storage more efficient. +3. The `geometry` column is WKB-encoded (the files are GeoParquet). + +## Release Discovery + +The [Overture STAC](https://stac.overturemaps.org/catalog.json) (SpatioTemporal Asset Catalog) is the authoritative source for discovering releases and determining which is the latest. + +The STAC also serves [latest.ddb](https://stac.overturemaps.org/latest.ddb), a DuckDB file with one view per feature type over the latest release. It is linked from the root `catalog.json`. Run `LOAD spatial;` before querying it. -### Release Discovery +For upcoming release dates, see the [release calendar](https://docs.overturemaps.org/release-calendar.json) ([schema](https://docs.overturemaps.org/release-calendar.schema.json)). -Use the [Overture STAC](https://stac.overturemaps.org/catalog.json) (SpatioTemporal Asset Catalog) for authoritative release data discovery and to determine the "latest" data. +## Feedback -For upcoming release dates, see the [docs.overturemaps.org/release-calendar.json](https://docs.overturemaps.org/release-calendar.json) ([schema](https://docs.overturemaps.org/release-calendar.schema.json)). +Share feedback on Overture data in [Discussions](https://github.com/OvertureMaps/data/discussions/new/choose). Tag the post with the release and theme (for example Places or Transportation) and include as much detail as possible. The relevant task force reviews each submission. -### Data Release Feedback +Feedback on the data schema belongs in the [schema repository's Discussions](https://github.com/OvertureMaps/schema/discussions). -We are very interested in feedback on the Overture data. Please use the [Discussion](https://github.com/OvertureMaps/data/discussions) section of this repo to comment. Tagging it with the release and relevant theme name (Places, Transportation) will help direct your ideas. Please include as much detail as possible. The associated Task Force will carefully review each submission and offer feedback where required. +Use the issue templates to report a [data bug](https://github.com/OvertureMaps/data/issues/new?template=data_bug.yml) or propose a [new data source](https://github.com/OvertureMaps/data/issues/new?template=new_data_source_proposal.yml). diff --git a/assets/overture-maps-foundation.png b/assets/overture-maps-foundation.png new file mode 100644 index 0000000..0dfd165 Binary files /dev/null and b/assets/overture-maps-foundation.png differ diff --git a/utils/fetch_releases_from_stac.py b/utils/fetch_releases_from_stac.py deleted file mode 100644 index dd6864d..0000000 --- a/utils/fetch_releases_from_stac.py +++ /dev/null @@ -1,84 +0,0 @@ -import json -import urllib.request -from urllib.parse import urlparse - -import duckdb - -STAC_CATALOG = "https://stac.overturemaps.org/catalog.json" -S3_BASE = "s3://overturemaps-us-west-2/release" -_USER_AGENT = "overturemaps-data/1.0" - -VIEWS = [ - ("address", "addresses", "address"), - ("bathymetry", "base", "bathymetry"), - ("building", "buildings", "building"), - ("building_part", "buildings", "building_part"), - ("connector", "transportation", "connector"), - ("division", "divisions", "division"), - ("division_area", "divisions", "division_area"), - ("division_boundary", "divisions", "division_boundary"), - ("infrastructure", "base", "infrastructure"), - ("land", "base", "land"), - ("land_cover", "base", "land_cover"), - ("land_use", "base", "land_use"), - ("place", "places", "place"), - ("segment", "transportation", "segment"), - ("water", "base", "water"), -] - - -def fetch_catalog(url: str, timeout: int = 30) -> dict: - req = urllib.request.Request(url, headers={"User-Agent": _USER_AGENT}) - with urllib.request.urlopen(req, timeout=timeout) as response: - return json.loads(response.read()) - - -def _release_id_from_href(href: str) -> str: - parts = [p for p in urlparse(href).path.split("/") if p and p != "."] - return parts[0] - - -def parse_releases(catalog: dict) -> dict: - latest = catalog["latest"] - releases = sorted( - [ - _release_id_from_href(link["href"]) - for link in catalog["links"] - if link["rel"] == "child" - ], - reverse=True, - ) - return {"latest": latest, "releases": releases} - - -def build_views_sql(latest: str, s3_base: str = S3_BASE) -> str: - stmts = ["INSTALL spatial;", "LOAD spatial;"] - for view_name, theme, type_ in VIEWS: - path = f"{s3_base}/{latest}/theme={theme}/type={type_}/*.parquet" - stmts.append( - f"CREATE OR REPLACE VIEW {view_name} AS (\n" - f" SELECT * FROM read_parquet('{path}')\n);" - ) - return "\n\n".join(stmts) - - -def create_duckdb_views(db_path: str, latest: str, s3_base: str = S3_BASE) -> None: - conn = duckdb.connect(db_path) - try: - conn.sql(build_views_sql(latest, s3_base)) - finally: - conn.close() - - -def main(): - catalog = fetch_catalog(STAC_CATALOG) - output = parse_releases(catalog) - - for release in output["releases"]: - print(f" - {release}") - - create_duckdb_views("latest.ddb", output["latest"]) - - -if __name__ == "__main__": - main() diff --git a/utils/requirements-test.txt b/utils/requirements-test.txt deleted file mode 100644 index 13f6026..0000000 --- a/utils/requirements-test.txt +++ /dev/null @@ -1,2 +0,0 @@ --r requirements.txt -pytest>=8.0.0 diff --git a/utils/requirements.txt b/utils/requirements.txt deleted file mode 100644 index 348ac20..0000000 --- a/utils/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -duckdb==1.3.2 diff --git a/utils/tests/__init__.py b/utils/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/utils/tests/test_fetch_releases_from_stac.py b/utils/tests/test_fetch_releases_from_stac.py deleted file mode 100644 index e482282..0000000 --- a/utils/tests/test_fetch_releases_from_stac.py +++ /dev/null @@ -1,199 +0,0 @@ -import json -import sys -import os -from unittest.mock import MagicMock, patch - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) - -from fetch_releases_from_stac import ( - VIEWS, - build_views_sql, - create_duckdb_views, - fetch_catalog, - parse_releases, -) - -SAMPLE_CATALOG = { - "type": "Catalog", - "id": "Overture Releases", - "stac_version": "1.1.0", - "description": "All Overture Releases", - "links": [ - {"rel": "root", "href": "./catalog.json", "type": "application/json"}, - { - "rel": "child", - "href": "./2026-05-20.0/catalog.json", - "type": "application/json", - "latest": True, - }, - { - "rel": "child", - "href": "./2026-04-15.0/catalog.json", - "type": "application/json", - }, - ], - "latest": "2026-05-20.0", -} - - -class TestFetchCatalog: - def test_returns_parsed_json(self): - mock_response = MagicMock() - mock_response.__enter__ = lambda s: s - mock_response.__exit__ = MagicMock(return_value=False) - mock_response.read.return_value = json.dumps(SAMPLE_CATALOG).encode() - - with patch("urllib.request.urlopen", return_value=mock_response): - result = fetch_catalog("https://stac.overturemaps.org/catalog.json") - - assert result == SAMPLE_CATALOG - - def test_uses_provided_url(self): - mock_response = MagicMock() - mock_response.__enter__ = lambda s: s - mock_response.__exit__ = MagicMock(return_value=False) - mock_response.read.return_value = json.dumps(SAMPLE_CATALOG).encode() - - with patch("urllib.request.urlopen", return_value=mock_response) as mock_open: - fetch_catalog("https://custom.example.com/catalog.json") - req = mock_open.call_args[0][0] - assert req.full_url == "https://custom.example.com/catalog.json" - - def test_applies_timeout(self): - mock_response = MagicMock() - mock_response.__enter__ = lambda s: s - mock_response.__exit__ = MagicMock(return_value=False) - mock_response.read.return_value = json.dumps(SAMPLE_CATALOG).encode() - - with patch("urllib.request.urlopen", return_value=mock_response) as mock_open: - fetch_catalog("https://stac.overturemaps.org/catalog.json", timeout=10) - assert mock_open.call_args[1]["timeout"] == 10 - - -class TestParseReleases: - def test_extracts_latest(self): - result = parse_releases(SAMPLE_CATALOG) - assert result["latest"] == "2026-05-20.0" - - def test_extracts_child_releases(self): - result = parse_releases(SAMPLE_CATALOG) - assert "2026-05-20.0" in result["releases"] - assert "2026-04-15.0" in result["releases"] - - def test_excludes_root_link(self): - result = parse_releases(SAMPLE_CATALOG) - # root link href is "./catalog.json" — split("/")[1] would be "catalog.json" - # but more importantly rel="root" should be excluded - assert "catalog.json" not in result["releases"] - assert len(result["releases"]) == 2 - - def test_releases_sorted_descending(self): - result = parse_releases(SAMPLE_CATALOG) - assert result["releases"] == sorted(result["releases"], reverse=True) - - def test_returns_dict_with_expected_keys(self): - result = parse_releases(SAMPLE_CATALOG) - assert set(result.keys()) == {"latest", "releases"} - - def test_empty_links(self): - catalog = {**SAMPLE_CATALOG, "links": [], "latest": "2026-05-20.0"} - result = parse_releases(catalog) - assert result["latest"] == "2026-05-20.0" - assert result["releases"] == [] - - def test_single_release(self): - catalog = { - **SAMPLE_CATALOG, - "links": [ - {"rel": "child", "href": "./2026-05-20.0/catalog.json"}, - ], - "latest": "2026-05-20.0", - } - result = parse_releases(catalog) - assert result["releases"] == ["2026-05-20.0"] - - def test_absolute_href_parsed_correctly(self): - catalog = { - **SAMPLE_CATALOG, - "links": [ - { - "rel": "child", - "href": "https://stac.overturemaps.org/2026-05-20.0/catalog.json", - }, - ], - "latest": "2026-05-20.0", - } - result = parse_releases(catalog) - assert result["releases"] == ["2026-05-20.0"] - - def test_relative_href_without_dotslash_parsed_correctly(self): - catalog = { - **SAMPLE_CATALOG, - "links": [ - {"rel": "child", "href": "2026-05-20.0/catalog.json"}, - ], - "latest": "2026-05-20.0", - } - result = parse_releases(catalog) - assert result["releases"] == ["2026-05-20.0"] - - - def test_contains_install_spatial(self): - sql = build_views_sql("2026-05-20.0") - assert "INSTALL spatial" in sql - - def test_contains_load_spatial(self): - sql = build_views_sql("2026-05-20.0") - assert "LOAD spatial" in sql - - def test_all_views_present(self): - sql = build_views_sql("2026-05-20.0") - for view_name, _, _ in VIEWS: - assert f"CREATE OR REPLACE VIEW {view_name}" in sql - - def test_latest_release_in_paths(self): - release = "2026-05-20.0" - sql = build_views_sql(release) - assert release in sql - - def test_custom_s3_base(self): - sql = build_views_sql("2026-05-20.0", s3_base="s3://my-bucket/release") - assert "s3://my-bucket/release" in sql - - def test_correct_theme_type_mapping(self): - sql = build_views_sql("2026-05-20.0") - assert "theme=addresses/type=address" in sql - assert "theme=buildings/type=building_part" in sql - assert "theme=transportation/type=segment" in sql - assert "theme=divisions/type=division_boundary" in sql - - def test_view_count_matches_views_constant(self): - sql = build_views_sql("2026-05-20.0") - count = sql.count("CREATE OR REPLACE VIEW") - assert count == len(VIEWS) - - -class TestCreateDuckdbViews: - def test_creates_all_views(self): - mock_conn = MagicMock() - with patch("duckdb.connect", return_value=mock_conn): - create_duckdb_views(":memory:", "2026-05-20.0") - mock_conn.sql.assert_called_once() - sql_arg = mock_conn.sql.call_args[0][0] - for view_name, _, _ in VIEWS: - assert f"CREATE OR REPLACE VIEW {view_name}" in sql_arg - - def test_views_reference_correct_release(self): - release = "2026-05-20.0" - mock_conn = MagicMock() - with patch("duckdb.connect", return_value=mock_conn): - create_duckdb_views(":memory:", release) - sql_arg = mock_conn.sql.call_args[0][0] - assert release in sql_arg - - def test_closes_connection(self): - mock_conn = MagicMock() - with patch("duckdb.connect", return_value=mock_conn): - create_duckdb_views(":memory:", "2026-05-20.0") - mock_conn.close.assert_called_once() -