Skip to content

Replace external TOPPRA with differentiable CPU/CUDA retiming #2565

Replace external TOPPRA with differentiable CPU/CUDA retiming

Replace external TOPPRA with differentiable CPU/CUDA retiming #2565

Workflow file for this run

name: CI/CD Pipeline
on:
push:
branches: [main]
tags:
- 'v*'
pull_request:
branches: [main, 'release/**']
schedule:
- cron: '0 2 * * *'
workflow_dispatch:
inputs:
test_mode:
description: 'Test selection mode for a manual run'
required: true
default: full
type: choice
options:
- partial
- full
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ !startsWith(github.ref, 'refs/tags/v') }}
permissions:
contents: read
pull-requests: read
jobs:
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: '3.11'
- name: Code Style check
run: |
echo "Workspace: ${GITHUB_WORKSPACE}"
ls
pip install black==26.3.1
black --check --diff --color ./
if [ $? -ne 0 ]; then
echo "Code style check failed, please run [black ./] before commit!"
exit 1
fi
- name: Public API docs check
run: python docs/scripts/check_api_docs.py
docs-changes:
runs-on: ubuntu-latest
outputs:
required: ${{ steps.filter.outputs.docs }}
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3
id: filter
with:
filters: |
docs:
- 'docs/**'
- 'embodichain/**'
- 'embodichain_tasks/**'
- 'examples/**'
- 'README.md'
- 'pyproject.toml'
- 'setup.py'
- 'VERSION'
test_plan:
name: Build test plan
if: >-
github.event_name == 'pull_request' ||
github.event_name == 'schedule' ||
github.event_name == 'workflow_dispatch'
runs-on: ubuntu-latest
outputs:
mode: ${{ steps.plan.outputs.mode }}
needs_tests: ${{ steps.plan.outputs.needs_tests }}
needs_docs: ${{ steps.plan.outputs.needs_docs }}
needs_hardware: ${{ steps.plan.outputs.needs_hardware }}
needs_gensim: ${{ steps.plan.outputs.needs_gensim }}
needs_curobo: ${{ steps.plan.outputs.needs_curobo }}
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
with:
fetch-depth: 0
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: '3.11'
- name: Install planner dependencies
shell: bash
run: |
python -m pip install --disable-pip-version-check pytest
python -m pip install --disable-pip-version-check pyyaml || \
echo 'PyYAML is unavailable; the manifest-only fallback remains enabled.'
- name: Validate impact analyzer
run: python -m pytest -q tests/ci
- name: Build changed-test plan
id: plan
shell: bash
env:
EVENT_NAME: ${{ github.event_name }}
MANUAL_MODE: ${{ inputs.test_mode || 'full' }}
PR_BASE: ${{ github.event.pull_request.base.sha }}
PR_HEAD: ${{ github.event.pull_request.head.sha }}
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
run: |
set -euo pipefail
if [[ "${EVENT_NAME}" == "pull_request" ]]; then
if [[ -z "${PR_BASE}" ]] || ! git cat-file -e "${PR_BASE}^{commit}" 2>/dev/null; then
python scripts/ci/select_tests.py \
--repo-root "${GITHUB_WORKSPACE}" \
--full \
--output test-plan.json
else
head_sha="${PR_HEAD:-HEAD}"
if [[ -z "${PR_HEAD}" ]] || ! git cat-file -e "${head_sha}^{commit}" 2>/dev/null; then
head_sha=HEAD
fi
# The event base can contain commits added to the target branch
# after the PR diverged. Diff from the common ancestor so the
# plan describes this PR's changes only.
merge_base="$(git merge-base "${PR_BASE}" "${head_sha}" 2>/dev/null || true)"
if [[ -z "${merge_base}" ]]; then
python scripts/ci/select_tests.py \
--repo-root "${GITHUB_WORKSPACE}" \
--full \
--output test-plan.json
else
python scripts/ci/select_tests.py \
--repo-root "${GITHUB_WORKSPACE}" \
--base "${merge_base}" \
--head "${head_sha}" \
--output test-plan.json
fi
fi
elif [[ "${EVENT_NAME}" == "workflow_dispatch" && "${MANUAL_MODE}" == "partial" ]]; then
base_sha="$(git merge-base "origin/${DEFAULT_BRANCH}" HEAD 2>/dev/null || true)"
if [[ -n "${base_sha}" ]]; then
python scripts/ci/select_tests.py \
--repo-root "${GITHUB_WORKSPACE}" \
--base "${base_sha}" \
--head HEAD \
--output test-plan.json
else
python scripts/ci/select_tests.py \
--repo-root "${GITHUB_WORKSPACE}" \
--full \
--output test-plan.json
fi
else
python scripts/ci/select_tests.py \
--repo-root "${GITHUB_WORKSPACE}" \
--full \
--output test-plan.json
fi
python - <<'PY' >> "${GITHUB_OUTPUT}"
import json
from pathlib import Path
plan = json.loads(Path("test-plan.json").read_text(encoding="utf-8"))
lanes = plan.get("lanes", {})
mode = str(plan.get("mode", "full-pr"))
slow_lanes = plan.get("slow_lanes", {})
if not isinstance(slow_lanes, dict):
slow_lanes = {}
all_lanes = {**lanes, **slow_lanes}
hardware = mode in {"full", "full-pr"} or any(
lane.removeprefix("slow-") in {"sim", "distributed", "gpu"}
for lane in all_lanes
)
print(f"mode={mode}")
print(f"needs_tests={str(mode != 'docs-only').lower()}")
print(
f"needs_docs={str(any(lane.endswith('docs') for lane in all_lanes)).lower()}"
)
print(f"needs_hardware={str(hardware).lower()}")
install = plan.get("install", {})
print(f"needs_gensim={str(bool(install.get('gensim'))).lower()}")
print(f"needs_curobo={str(bool(install.get('curobo'))).lower()}")
PY
- name: Upload test plan
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: test-plan
path: test-plan.json
if-no-files-found: error
- name: Publish test plan summary
shell: bash
run: |
python - <<'PY' >> "${GITHUB_STEP_SUMMARY}"
import json
from pathlib import Path
plan = json.loads(Path("test-plan.json").read_text(encoding="utf-8"))
lanes = plan.get("lanes", {})
slow_lanes = plan.get("slow_lanes", {})
if not isinstance(slow_lanes, dict):
slow_lanes = {}
all_lanes = {**lanes, **slow_lanes}
print("## Test impact plan")
print()
print(f"- Mode: `{plan.get('mode')}`")
print(f"- Risk: `{plan.get('risk')}`")
print(f"- Changed paths: {len(plan.get('changed', []))}")
print(f"- Selectors: {len(plan.get('selectors', []))}")
hardware = plan.get("mode") in {"full", "full-pr"} or any(
lane.removeprefix("slow-") in {"sim", "distributed", "gpu"}
for lane in all_lanes
)
print(f"- Hardware lanes: `{str(hardware).lower()}`")
print(f"- Fallback: `{plan.get('fallback_reason') or 'none'}`")
print()
print("| Lane | Selectors |")
print("| --- | ---: |")
for lane, selectors in all_lanes.items():
print(f"| {lane} | {len(selectors)} |")
PY
build:
if: ${{ !startsWith(github.ref, 'refs/tags/v') && needs.docs-changes.outputs.required == 'true' }}
needs: [lint, docs-changes]
runs-on: Linux
env:
NVIDIA_DRIVER_CAPABILITIES: all
NVIDIA_VISIBLE_DEVICES: all
NVIDIA_DISABLE_REQUIRE: 1
container: &container_template
image: 192.168.3.13:5000/dexsdk:ubuntu22.04-cuda12.8.0-h5ffmpeg-v3-py311
volumes:
- "/cache:/cache"
- "/usr/share/vulkan/icd.d:/usr/share/vulkan/icd.d"
- "/usr/share/vulkan/implicit_layer.d:/usr/share/vulkan/implicit_layer.d"
- "/usr/share/glvnd/egl_vendor.d:/usr/share/glvnd/egl_vendor.d"
- "/tmp/.X11-unix:/tmp/.X11-unix"
- "/dev/shm/shared:/dev/shm/shared"
- "/usr/share/nvidia:/usr/share/nvidia"
options: --memory 100g --gpus device=0 --shm-size 53687091200
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- uses: ./.github/actions/activate-conda
- name: Prepare Git for architecture generation
shell: bash
run: |
git config --global --add safe.directory "$GITHUB_WORKSPACE"
git -C "$GITHUB_WORKSPACE" rev-parse \
--verify --end-of-options 'HEAD^{commit}'
# Restore the full multi-version site from the last successful build.
# The key never matches exactly (run_id is unique), so restore-keys is
# always used to pick up the most recently saved full site.
- name: Restore full multi-version docs site
if: github.event_name == 'push'
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: docs/build/html
key: docs-full-site-${{ github.repository }}-${{ github.run_id }}
restore-keys: |
docs-full-site-${{ github.repository }}-
# Tag-scoped caches are invisible on main; merge live Pages so releases survive.
- name: Merge versions from live GitHub Pages
if: github.event_name == 'push'
shell: bash
run: |
SITE_URL="https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }}"
python3 ${GITHUB_WORKSPACE}/docs/scripts/merge_published_site.py \
--build-dir ${GITHUB_WORKSPACE}/docs/build/html \
--site-base-url "${SITE_URL}" \
--skip-version main
- name: Set up architecture frontend toolchain
if: hashFiles('docs/scripts/build_architecture_assets.py') != ''
uses: actions/setup-node@v4
with:
node-version: '22'
cache: npm
cache-dependency-path: docs/architecture/web/package-lock.json
- name: Install architecture frontend dependencies
if: hashFiles('docs/scripts/build_architecture_assets.py') != ''
run: npm --prefix docs/architecture/web ci
- name: Build docs
shell: bash
run: |
pip install -e ".[gensim]" \
--extra-index-url http://pyp.open3dv.site:2345/simple/ \
--trusted-host pyp.open3dv.site \
--extra-index-url https://download.blender.org/pypi/
pip install -r docs/requirements.txt
python3 docs/scripts/sync_readme.py
cd ${GITHUB_WORKSPACE}/docs
pip uninstall pymeshlab -y
pip install pymeshlab==2023.12.post3
echo "Building dev docs for main branch..."
# Only rebuild main/ — other versions come from cache + live Pages merge
rm -rf build/html/main
sphinx-build source build/html/main
cd build/html
# Regenerate versions.json and root index.html from all present dirs
python3 ${GITHUB_WORKSPACE}/docs/scripts/generate_versions_json.py \
--build-dir .
# Default-branch cache only (tag-scoped caches are not visible on main).
- name: Save full multi-version docs site
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: docs/build/html
key: docs-full-site-${{ github.repository }}-${{ github.run_id }}
- name: Upload docs site artifact
if: github.event_name == 'push'
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: docs-site
path: ${{ github.workspace }}/docs/build/html
test:
if: >-
(github.event_name == 'pull_request' ||
github.event_name == 'schedule' ||
github.event_name == 'workflow_dispatch') &&
needs.test_plan.outputs.needs_tests == 'true'
needs: [lint, test_plan]
runs-on: Linux
env:
NVIDIA_DRIVER_CAPABILITIES: all
NVIDIA_VISIBLE_DEVICES: all
NVIDIA_DISABLE_REQUIRE: 1
HF_ENDPOINT: https://hf-mirror.com
PIP_CACHE_DIR: /cache/pip
NEEDS_GENSIM: ${{ needs.test_plan.outputs.needs_gensim }}
NEEDS_CUROBO: ${{ needs.test_plan.outputs.needs_curobo }}
container: *container_template
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- uses: ./.github/actions/activate-conda
- name: Download test plan
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
name: test-plan
path: .
- name: Install test package
shell: bash -l {0}
run: |
source activate py311
set -euo pipefail
if [[ "${NEEDS_GENSIM}" == "true" ]]; then
package='.[gensim]'
else
package='.'
fi
pip install -e "${package}" \
--extra-index-url http://pyp.open3dv.site:2345/simple/ \
--trusted-host pyp.open3dv.site \
--extra-index-url https://download.blender.org/pypi/
if [[ "${NEEDS_CUROBO}" == "true" ]]; then
pip install \
"nvidia-curobo[cu12] @ git+https://github.com/NVlabs/curobo.git@v0.8.0" \
--extra-index-url http://pyp.open3dv.site:2345/simple/ \
--trusted-host pyp.open3dv.site \
--extra-index-url https://download.blender.org/pypi/
fi
pip install pytest-xdist
- name: Verify Scene Engine gensim installation
if: env.NEEDS_GENSIM == 'true'
run: |
python -c "import matplotlib, numpy, open3d, requests, scipy, shapely, trimesh; from PIL import Image; import embodichain.gen_sim.scene_engine.pipeline.generate"
- name: Run selected tests
shell: bash -l {0}
run: |
source activate py311
# Run GPU processes before the long DexSim lane. Native Vulkan/CUDA
# teardown can briefly retain allocations after the sim subprocess;
# this order prevents the GPU lane from inheriting that pressure.
python scripts/ci/run_test_plan.py \
--plan test-plan.json \
--repo-root "${GITHUB_WORKSPACE}" \
--lane fast \
--lane distributed \
--lane gpu \
--lane sim
test_docs:
name: Documentation tests
if: >-
(github.event_name == 'pull_request' ||
github.event_name == 'schedule' ||
github.event_name == 'workflow_dispatch') &&
needs.test_plan.outputs.needs_docs == 'true'
needs: [lint, test_plan]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
with:
fetch-depth: 0
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: '3.11'
- name: Install docs test dependencies
run: python -m pip install --disable-pip-version-check pytest -r docs/requirements.txt
- name: Download test plan
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
name: test-plan
path: .
- name: Run documentation tests
run: python scripts/ci/run_test_plan.py --plan test-plan.json --repo-root "${GITHUB_WORKSPACE}" --lane docs
- name: Set up architecture frontend toolchain
if: hashFiles('docs/scripts/build_architecture_assets.py') != ''
uses: actions/setup-node@v4
with:
node-version: '22'
cache: npm
cache-dependency-path: docs/architecture/web/package-lock.json
- name: Install architecture frontend dependencies
if: hashFiles('docs/scripts/build_architecture_assets.py') != ''
run: npm --prefix docs/architecture/web ci
- name: Verify architecture frontend and Sphinx integration
working-directory: docs/architecture/web
run: |
npx playwright install --with-deps chromium
npm run data
npm test
npm run test:browser
npm run test:docs
npm run format:check
test_gate:
name: Test gate
if: >-
always() &&
(github.event_name == 'pull_request' ||
github.event_name == 'schedule' ||
github.event_name == 'workflow_dispatch')
needs: [lint, test_plan, test, test_docs]
runs-on: ubuntu-latest
steps:
- name: Validate selected test result
env:
LINT_RESULT: ${{ needs.lint.result }}
PLAN_RESULT: ${{ needs.test_plan.result }}
TEST_RESULT: ${{ needs.test.result }}
DOCS_RESULT: ${{ needs.test_docs.result }}
DOCS_NEEDED: ${{ needs.test_plan.outputs.needs_docs }}
TESTS_NEEDED: ${{ needs.test_plan.outputs.needs_tests }}
shell: bash
run: |
set -euo pipefail
if [[ "${LINT_RESULT}" != "success" ]]; then
echo "lint result: ${LINT_RESULT}"
exit 1
fi
if [[ "${PLAN_RESULT}" != "success" ]]; then
echo "test-plan result: ${PLAN_RESULT}"
exit 1
fi
if [[ "${DOCS_NEEDED}" == "true" && "${DOCS_RESULT}" != "success" ]]; then
echo "documentation test result: ${DOCS_RESULT}"
exit 1
fi
if [[ "${TESTS_NEEDED}" == "true" && "${TEST_RESULT}" != "success" ]]; then
echo "selected test result: ${TEST_RESULT}"
exit 1
fi
if [[ "${TESTS_NEEDED}" != "true" && "${TEST_RESULT}" != "skipped" ]]; then
echo "unexpected selected test result: ${TEST_RESULT}"
exit 1
fi
release-build:
if: startsWith(github.ref, 'refs/tags/v')
needs: lint
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: '3.11'
- name: Validate release versions
shell: bash
run: |
release_version="${GITHUB_REF_NAME#v}"
core_version="$(tr -d '[:space:]' < VERSION)"
if [[ "${core_version}" != "${release_version}" ]]; then
echo "VERSION=${core_version} does not match tag ${GITHUB_REF_NAME}."
exit 1
fi
- name: Build and validate distributions
run: |
python -m pip install --upgrade pip
python -m pip install build packaging twine
python -m build
python -m twine check dist/*
- name: Validate PyPI dependency metadata
run: python scripts/validate_wheel_metadata.py dist/*.whl
- name: Validate wheel contents
shell: bash
run: |
python - <<'PY'
from email.parser import Parser
from pathlib import Path
from zipfile import ZipFile
version = Path("VERSION").read_text(encoding="utf-8").strip()
wheels = list(Path("dist").glob(f"embodichain-{version}-*.whl"))
if len(wheels) != 1:
raise SystemExit(f"Expected one EmbodiChain wheel, found: {wheels}")
dist_info = f"embodichain-{version}.dist-info/"
required = {
"embodichain/VERSION",
"embodichain/gen_sim/simready_pipeline/configs/gen_config.json",
"embodichain_tasks/__init__.py",
"embodichain_tasks/configs/tasks/manipulation/tableware/pour_water/env.yaml",
}
with ZipFile(wheels[0]) as wheel:
members = set(wheel.namelist())
entry_points = wheel.read(f"{dist_info}entry_points.txt").decode()
metadata = Parser().parsestr(wheel.read(f"{dist_info}METADATA").decode())
runtime_version = wheel.read("embodichain/VERSION").decode().strip()
if metadata["Version"] != version or runtime_version != version:
raise SystemExit(
f"Wheel version mismatch: expected={version}, "
f"metadata={metadata['Version']}, runtime={runtime_version}"
)
source_task_configs = {
path.as_posix()
for path in Path("embodichain_tasks/configs").rglob("*")
if path.suffix in {".json", ".yaml", ".yml"}
}
wheel_task_configs = {
member
for member in members
if member.startswith("embodichain_tasks/configs/")
and member.endswith((".json", ".yaml", ".yml"))
}
unexpected = sorted(
member
for member in members
if not member.startswith(
("embodichain/", "embodichain_tasks/", dist_info)
)
or "__pycache__" in member
or member.endswith((".pyc", ".pyo"))
)
missing = sorted(required - members)
missing_task_configs = sorted(source_task_configs - wheel_task_configs)
unexpected_task_configs = sorted(wheel_task_configs - source_task_configs)
task_entry_point = "embodichain_tasks = embodichain_tasks"
if (
unexpected
or missing
or missing_task_configs
or unexpected_task_configs
or task_entry_point not in entry_points
):
raise SystemExit(
"Invalid wheel contents; "
f"unexpected={unexpected}, missing={missing}, "
f"missing_task_configs={missing_task_configs}, "
f"unexpected_task_configs={unexpected_task_configs}, "
f"task_entry_point={task_entry_point in entry_points}"
)
PY
- name: Upload validated distributions
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: python-distributions
path: dist/