mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-07-19 00:19:02 +00:00
* feat(infra): add runtime package support for skills Install nodejs, npm, pandoc, github-cli + pre-install Python packages (openpyxl, pandas, python-pptx, markitdown) and Node packages (docx, pptxgenjs). Configure runtime dirs for agent pip/npm installs with PIP_TARGET, NPM_CONFIG_PREFIX, NODE_PATH to enable dynamic package installation in read-only container environment. * feat(infra): add bundled skills with runtime package support - Add 5 bundled skills: docx, pdf, pptx, xlsx, skill-creator from container skills-store - Wire GOCLAW_BUILTIN_SKILLS_DIR env var in gateway and CLI - Support optional runtime packages alongside dynamic skill loading - Update Dockerfile to COPY bundled-skills at /app/bundled-skills/ - Add PIP_CACHE_DIR in docker-entrypoint.sh for clean pip installs - Document bundled skills in 14-skills-runtime.md section 6 * feat(infra): remove ai-multimodal skill directory from bundled skills Remove the ai-multimodal skill package as part of consolidating runtime package support for bundled skills. This directory is no longer needed in the bundled skills structure. * feat(ci): add semantic release and Docker Hub publishing Add go-semantic-release workflow to auto-create semver tags on merge to main. Extend docker-publish to push all variants to both GHCR and Docker Hub (digitop/goclaw). * feat(skills): add system skills infrastructure with is_system column, dep scanning, and seeder - Migration 000017: add is_system boolean column with partial index - Store layer: UpsertSystemSkill, delete protection, IsSystemSkill - ListAccessible auto-includes system skills (no grants needed) - ListWithGrantStatus returns is_system field - Dependency scanner: auto-detect deps from scripts/ or skill-manifest.json - Dependency checker: verify system binaries, Python/Node packages - Seeder: seed bundled skills into DB on startup (idempotent via hash) - Gateway wiring: GOCLAW_BUNDLED_SKILLS_DIR env for bundled skills - HTTP: delete guard (403), slug conflict check (409), rescan-deps endpoint - UI: System badge, hide delete for system skills, rescan deps button - Agent skills tab: "Always available" for system skills - i18n: en/vi/zh keys for system skills, deps scanning * feat(skills): conditional system prompt, skill manifests, and Zip Slip fix - System prompt: only show package list when python3/node are available - Add skill-manifest.json for pdf, docx, xlsx, pptx bundled skills - Fix Zip Slip vulnerability in office/unpack.py (all 3 copies) * refactor(skills): extract shared office code to _shared/ and deduplicate Move office scripts (pack, unpack, validate, schemas, validators) from duplicated copies in docx/xlsx/pptx to skills/_shared/office/ with symlinks. Remove soffice.py (non-functional in containers) and update SKILL.md references to use soffice binary directly. Update seeder copyDir to follow symlinks. Removes ~45K lines of duplicate code across 3 skills. * fix(skills): address code review findings for system skills integration - H1: Remove dead symlink branch in copyDir (filepath.Walk follows symlinks) - H3: Fix rescan-deps to query ALL skills (including archived) and re-activate when deps become available; add ListAllSkills() + Status field to SkillInfo - H4: Add Status field to SkillCreateParams, stop overloading Visibility - M1: Batch Python/Node dep checks into single subprocess per runtime - M4: Add rows.Err() check in ListSkills to prevent caching partial results * feat(skills): async dep checking with realtime WS events Split Seed() into sync DB upsert + async CheckDepsAsync() goroutine. Gateway startup no longer blocks on Python/Node subprocess dep checks. - Seed() returns seeded skills list, all initially status="active" - CheckDepsAsync() runs in background, emits skill.deps.checked per-skill - skill.deps.complete event emitted when all checks finish - Each failed dep check: archives skill + BumpVersion() for immediate cache invalidation so next agent turn picks up the change - UI: use-query-invalidation listens to skill.deps.* events → auto-refresh skills list in realtime * feat(skills): system skills integration with toggle, dep checking, and per-item install - Add is_system, deps, enabled columns to skills table (migration 017) - Seed bundled core skills (pdf, docx, pptx, xlsx, skill-creator) on startup - PYTHONPATH-based dep detection — eliminates false positives from local modules - Per-item dep install UI with individual status (installing/success/error) - Enable/disable toggle for core and custom skills (independent of dep status) - Re-run dep check when skill is toggled back on - Inline skill thresholds: 40 skills / 5000 tokens before switching to search mode - Fix UpsertSystemSkill: backfill null file_hash without bumping DB version - Remove redundant skill-manifest.json files (replaced by deps JSONB column) - Show author from frontmatter in custom skills tab - Runtime checker for python3/pip3/node/npm availability - WS events for dep checking/installing progress - docs: add 15-core-skills-system.md, 16-skill-publishing.md --------- Co-authored-by: Goon <duy@wearetopgroup.com>
137 lines
4.2 KiB
Python
137 lines
4.2 KiB
Python
"""Unpack Office files (DOCX, PPTX, XLSX) for editing.
|
|
|
|
Extracts the ZIP archive, pretty-prints XML files, and optionally:
|
|
- Merges adjacent runs with identical formatting (DOCX only)
|
|
- Simplifies adjacent tracked changes from same author (DOCX only)
|
|
|
|
Usage:
|
|
python unpack.py <office_file> <output_dir> [options]
|
|
|
|
Examples:
|
|
python unpack.py document.docx unpacked/
|
|
python unpack.py presentation.pptx unpacked/
|
|
python unpack.py document.docx unpacked/ --merge-runs false
|
|
"""
|
|
|
|
import argparse
|
|
import sys
|
|
import zipfile
|
|
from pathlib import Path
|
|
|
|
import defusedxml.minidom
|
|
|
|
from helpers.merge_runs import merge_runs as do_merge_runs
|
|
from helpers.simplify_redlines import simplify_redlines as do_simplify_redlines
|
|
|
|
SMART_QUOTE_REPLACEMENTS = {
|
|
"\u201c": "“",
|
|
"\u201d": "”",
|
|
"\u2018": "‘",
|
|
"\u2019": "’",
|
|
}
|
|
|
|
|
|
def unpack(
|
|
input_file: str,
|
|
output_directory: str,
|
|
merge_runs: bool = True,
|
|
simplify_redlines: bool = True,
|
|
) -> tuple[None, str]:
|
|
input_path = Path(input_file)
|
|
output_path = Path(output_directory)
|
|
suffix = input_path.suffix.lower()
|
|
|
|
if not input_path.exists():
|
|
return None, f"Error: {input_file} does not exist"
|
|
|
|
if suffix not in {".docx", ".pptx", ".xlsx"}:
|
|
return None, f"Error: {input_file} must be a .docx, .pptx, or .xlsx file"
|
|
|
|
try:
|
|
output_path.mkdir(parents=True, exist_ok=True)
|
|
|
|
with zipfile.ZipFile(input_path, "r") as zf:
|
|
for info in zf.infolist():
|
|
target = (output_path / info.filename).resolve()
|
|
if not str(target).startswith(str(output_path.resolve())):
|
|
raise ValueError(f"Zip entry escapes target: {info.filename}")
|
|
zf.extractall(output_path)
|
|
|
|
xml_files = list(output_path.rglob("*.xml")) + list(output_path.rglob("*.rels"))
|
|
for xml_file in xml_files:
|
|
_pretty_print_xml(xml_file)
|
|
|
|
message = f"Unpacked {input_file} ({len(xml_files)} XML files)"
|
|
|
|
if suffix == ".docx":
|
|
if simplify_redlines:
|
|
simplify_count, _ = do_simplify_redlines(str(output_path))
|
|
message += f", simplified {simplify_count} tracked changes"
|
|
|
|
if merge_runs:
|
|
merge_count, _ = do_merge_runs(str(output_path))
|
|
message += f", merged {merge_count} runs"
|
|
|
|
for xml_file in xml_files:
|
|
_escape_smart_quotes(xml_file)
|
|
|
|
return None, message
|
|
|
|
except zipfile.BadZipFile:
|
|
return None, f"Error: {input_file} is not a valid Office file"
|
|
except Exception as e:
|
|
return None, f"Error unpacking: {e}"
|
|
|
|
|
|
def _pretty_print_xml(xml_file: Path) -> None:
|
|
try:
|
|
content = xml_file.read_text(encoding="utf-8")
|
|
dom = defusedxml.minidom.parseString(content)
|
|
xml_file.write_bytes(dom.toprettyxml(indent=" ", encoding="utf-8"))
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def _escape_smart_quotes(xml_file: Path) -> None:
|
|
try:
|
|
content = xml_file.read_text(encoding="utf-8")
|
|
for char, entity in SMART_QUOTE_REPLACEMENTS.items():
|
|
content = content.replace(char, entity)
|
|
xml_file.write_text(content, encoding="utf-8")
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser(
|
|
description="Unpack an Office file (DOCX, PPTX, XLSX) for editing"
|
|
)
|
|
parser.add_argument("input_file", help="Office file to unpack")
|
|
parser.add_argument("output_directory", help="Output directory")
|
|
parser.add_argument(
|
|
"--merge-runs",
|
|
type=lambda x: x.lower() == "true",
|
|
default=True,
|
|
metavar="true|false",
|
|
help="Merge adjacent runs with identical formatting (DOCX only, default: true)",
|
|
)
|
|
parser.add_argument(
|
|
"--simplify-redlines",
|
|
type=lambda x: x.lower() == "true",
|
|
default=True,
|
|
metavar="true|false",
|
|
help="Merge adjacent tracked changes from same author (DOCX only, default: true)",
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
_, message = unpack(
|
|
args.input_file,
|
|
args.output_directory,
|
|
merge_runs=args.merge_runs,
|
|
simplify_redlines=args.simplify_redlines,
|
|
)
|
|
print(message)
|
|
|
|
if "Error" in message:
|
|
sys.exit(1)
|