mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-07-13 17:09:53 +00:00
ace07509b7
* feat(infra): add runtime package support for skills Install nodejs, npm, pandoc, github-cli + pre-install Python packages (openpyxl, pandas, python-pptx, markitdown) and Node packages (docx, pptxgenjs). Configure runtime dirs for agent pip/npm installs with PIP_TARGET, NPM_CONFIG_PREFIX, NODE_PATH to enable dynamic package installation in read-only container environment. * feat(infra): add bundled skills with runtime package support - Add 5 bundled skills: docx, pdf, pptx, xlsx, skill-creator from container skills-store - Wire GOCLAW_BUILTIN_SKILLS_DIR env var in gateway and CLI - Support optional runtime packages alongside dynamic skill loading - Update Dockerfile to COPY bundled-skills at /app/bundled-skills/ - Add PIP_CACHE_DIR in docker-entrypoint.sh for clean pip installs - Document bundled skills in 14-skills-runtime.md section 6 * feat(infra): remove ai-multimodal skill directory from bundled skills Remove the ai-multimodal skill package as part of consolidating runtime package support for bundled skills. This directory is no longer needed in the bundled skills structure. * feat(ci): add semantic release and Docker Hub publishing Add go-semantic-release workflow to auto-create semver tags on merge to main. Extend docker-publish to push all variants to both GHCR and Docker Hub (digitop/goclaw). * feat(skills): add system skills infrastructure with is_system column, dep scanning, and seeder - Migration 000017: add is_system boolean column with partial index - Store layer: UpsertSystemSkill, delete protection, IsSystemSkill - ListAccessible auto-includes system skills (no grants needed) - ListWithGrantStatus returns is_system field - Dependency scanner: auto-detect deps from scripts/ or skill-manifest.json - Dependency checker: verify system binaries, Python/Node packages - Seeder: seed bundled skills into DB on startup (idempotent via hash) - Gateway wiring: GOCLAW_BUNDLED_SKILLS_DIR env for bundled skills - HTTP: delete guard (403), slug conflict check (409), rescan-deps endpoint - UI: System badge, hide delete for system skills, rescan deps button - Agent skills tab: "Always available" for system skills - i18n: en/vi/zh keys for system skills, deps scanning * feat(skills): conditional system prompt, skill manifests, and Zip Slip fix - System prompt: only show package list when python3/node are available - Add skill-manifest.json for pdf, docx, xlsx, pptx bundled skills - Fix Zip Slip vulnerability in office/unpack.py (all 3 copies) * refactor(skills): extract shared office code to _shared/ and deduplicate Move office scripts (pack, unpack, validate, schemas, validators) from duplicated copies in docx/xlsx/pptx to skills/_shared/office/ with symlinks. Remove soffice.py (non-functional in containers) and update SKILL.md references to use soffice binary directly. Update seeder copyDir to follow symlinks. Removes ~45K lines of duplicate code across 3 skills. * fix(skills): address code review findings for system skills integration - H1: Remove dead symlink branch in copyDir (filepath.Walk follows symlinks) - H3: Fix rescan-deps to query ALL skills (including archived) and re-activate when deps become available; add ListAllSkills() + Status field to SkillInfo - H4: Add Status field to SkillCreateParams, stop overloading Visibility - M1: Batch Python/Node dep checks into single subprocess per runtime - M4: Add rows.Err() check in ListSkills to prevent caching partial results * feat(skills): async dep checking with realtime WS events Split Seed() into sync DB upsert + async CheckDepsAsync() goroutine. Gateway startup no longer blocks on Python/Node subprocess dep checks. - Seed() returns seeded skills list, all initially status="active" - CheckDepsAsync() runs in background, emits skill.deps.checked per-skill - skill.deps.complete event emitted when all checks finish - Each failed dep check: archives skill + BumpVersion() for immediate cache invalidation so next agent turn picks up the change - UI: use-query-invalidation listens to skill.deps.* events → auto-refresh skills list in realtime * feat(skills): system skills integration with toggle, dep checking, and per-item install - Add is_system, deps, enabled columns to skills table (migration 017) - Seed bundled core skills (pdf, docx, pptx, xlsx, skill-creator) on startup - PYTHONPATH-based dep detection — eliminates false positives from local modules - Per-item dep install UI with individual status (installing/success/error) - Enable/disable toggle for core and custom skills (independent of dep status) - Re-run dep check when skill is toggled back on - Inline skill thresholds: 40 skills / 5000 tokens before switching to search mode - Fix UpsertSystemSkill: backfill null file_hash without bumping DB version - Remove redundant skill-manifest.json files (replaced by deps JSONB column) - Show author from frontmatter in custom skills tab - Runtime checker for python3/pip3/node/npm availability - WS events for dep checking/installing progress - docs: add 15-core-skills-system.md, 16-skill-publishing.md --------- Co-authored-by: Goon <duy@wearetopgroup.com>
198 lines
5.6 KiB
Python
198 lines
5.6 KiB
Python
"""Simplify tracked changes by merging adjacent w:ins or w:del elements.
|
|
|
|
Merges adjacent <w:ins> elements from the same author into a single element.
|
|
Same for <w:del> elements. This makes heavily-redlined documents easier to
|
|
work with by reducing the number of tracked change wrappers.
|
|
|
|
Rules:
|
|
- Only merges w:ins with w:ins, w:del with w:del (same element type)
|
|
- Only merges if same author (ignores timestamp differences)
|
|
- Only merges if truly adjacent (only whitespace between them)
|
|
"""
|
|
|
|
import xml.etree.ElementTree as ET
|
|
import zipfile
|
|
from pathlib import Path
|
|
|
|
import defusedxml.minidom
|
|
|
|
WORD_NS = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"
|
|
|
|
|
|
def simplify_redlines(input_dir: str) -> tuple[int, str]:
|
|
doc_xml = Path(input_dir) / "word" / "document.xml"
|
|
|
|
if not doc_xml.exists():
|
|
return 0, f"Error: {doc_xml} not found"
|
|
|
|
try:
|
|
dom = defusedxml.minidom.parseString(doc_xml.read_text(encoding="utf-8"))
|
|
root = dom.documentElement
|
|
|
|
merge_count = 0
|
|
|
|
containers = _find_elements(root, "p") + _find_elements(root, "tc")
|
|
|
|
for container in containers:
|
|
merge_count += _merge_tracked_changes_in(container, "ins")
|
|
merge_count += _merge_tracked_changes_in(container, "del")
|
|
|
|
doc_xml.write_bytes(dom.toxml(encoding="UTF-8"))
|
|
return merge_count, f"Simplified {merge_count} tracked changes"
|
|
|
|
except Exception as e:
|
|
return 0, f"Error: {e}"
|
|
|
|
|
|
def _merge_tracked_changes_in(container, tag: str) -> int:
|
|
merge_count = 0
|
|
|
|
tracked = [
|
|
child
|
|
for child in container.childNodes
|
|
if child.nodeType == child.ELEMENT_NODE and _is_element(child, tag)
|
|
]
|
|
|
|
if len(tracked) < 2:
|
|
return 0
|
|
|
|
i = 0
|
|
while i < len(tracked) - 1:
|
|
curr = tracked[i]
|
|
next_elem = tracked[i + 1]
|
|
|
|
if _can_merge_tracked(curr, next_elem):
|
|
_merge_tracked_content(curr, next_elem)
|
|
container.removeChild(next_elem)
|
|
tracked.pop(i + 1)
|
|
merge_count += 1
|
|
else:
|
|
i += 1
|
|
|
|
return merge_count
|
|
|
|
|
|
def _is_element(node, tag: str) -> bool:
|
|
name = node.localName or node.tagName
|
|
return name == tag or name.endswith(f":{tag}")
|
|
|
|
|
|
def _get_author(elem) -> str:
|
|
author = elem.getAttribute("w:author")
|
|
if not author:
|
|
for attr in elem.attributes.values():
|
|
if attr.localName == "author" or attr.name.endswith(":author"):
|
|
return attr.value
|
|
return author
|
|
|
|
|
|
def _can_merge_tracked(elem1, elem2) -> bool:
|
|
if _get_author(elem1) != _get_author(elem2):
|
|
return False
|
|
|
|
node = elem1.nextSibling
|
|
while node and node != elem2:
|
|
if node.nodeType == node.ELEMENT_NODE:
|
|
return False
|
|
if node.nodeType == node.TEXT_NODE and node.data.strip():
|
|
return False
|
|
node = node.nextSibling
|
|
|
|
return True
|
|
|
|
|
|
def _merge_tracked_content(target, source):
|
|
while source.firstChild:
|
|
child = source.firstChild
|
|
source.removeChild(child)
|
|
target.appendChild(child)
|
|
|
|
|
|
def _find_elements(root, tag: str) -> list:
|
|
results = []
|
|
|
|
def traverse(node):
|
|
if node.nodeType == node.ELEMENT_NODE:
|
|
name = node.localName or node.tagName
|
|
if name == tag or name.endswith(f":{tag}"):
|
|
results.append(node)
|
|
for child in node.childNodes:
|
|
traverse(child)
|
|
|
|
traverse(root)
|
|
return results
|
|
|
|
|
|
def get_tracked_change_authors(doc_xml_path: Path) -> dict[str, int]:
|
|
if not doc_xml_path.exists():
|
|
return {}
|
|
|
|
try:
|
|
tree = ET.parse(doc_xml_path)
|
|
root = tree.getroot()
|
|
except ET.ParseError:
|
|
return {}
|
|
|
|
namespaces = {"w": WORD_NS}
|
|
author_attr = f"{{{WORD_NS}}}author"
|
|
|
|
authors: dict[str, int] = {}
|
|
for tag in ["ins", "del"]:
|
|
for elem in root.findall(f".//w:{tag}", namespaces):
|
|
author = elem.get(author_attr)
|
|
if author:
|
|
authors[author] = authors.get(author, 0) + 1
|
|
|
|
return authors
|
|
|
|
|
|
def _get_authors_from_docx(docx_path: Path) -> dict[str, int]:
|
|
try:
|
|
with zipfile.ZipFile(docx_path, "r") as zf:
|
|
if "word/document.xml" not in zf.namelist():
|
|
return {}
|
|
with zf.open("word/document.xml") as f:
|
|
tree = ET.parse(f)
|
|
root = tree.getroot()
|
|
|
|
namespaces = {"w": WORD_NS}
|
|
author_attr = f"{{{WORD_NS}}}author"
|
|
|
|
authors: dict[str, int] = {}
|
|
for tag in ["ins", "del"]:
|
|
for elem in root.findall(f".//w:{tag}", namespaces):
|
|
author = elem.get(author_attr)
|
|
if author:
|
|
authors[author] = authors.get(author, 0) + 1
|
|
return authors
|
|
except (zipfile.BadZipFile, ET.ParseError):
|
|
return {}
|
|
|
|
|
|
def infer_author(modified_dir: Path, original_docx: Path, default: str = "Claude") -> str:
|
|
modified_xml = modified_dir / "word" / "document.xml"
|
|
modified_authors = get_tracked_change_authors(modified_xml)
|
|
|
|
if not modified_authors:
|
|
return default
|
|
|
|
original_authors = _get_authors_from_docx(original_docx)
|
|
|
|
new_changes: dict[str, int] = {}
|
|
for author, count in modified_authors.items():
|
|
original_count = original_authors.get(author, 0)
|
|
diff = count - original_count
|
|
if diff > 0:
|
|
new_changes[author] = diff
|
|
|
|
if not new_changes:
|
|
return default
|
|
|
|
if len(new_changes) == 1:
|
|
return next(iter(new_changes))
|
|
|
|
raise ValueError(
|
|
f"Multiple authors added new changes: {new_changes}. "
|
|
"Cannot infer which author to validate."
|
|
)
|