mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-06-11 22:11:49 +00:00
ace07509b7
* feat(infra): add runtime package support for skills Install nodejs, npm, pandoc, github-cli + pre-install Python packages (openpyxl, pandas, python-pptx, markitdown) and Node packages (docx, pptxgenjs). Configure runtime dirs for agent pip/npm installs with PIP_TARGET, NPM_CONFIG_PREFIX, NODE_PATH to enable dynamic package installation in read-only container environment. * feat(infra): add bundled skills with runtime package support - Add 5 bundled skills: docx, pdf, pptx, xlsx, skill-creator from container skills-store - Wire GOCLAW_BUILTIN_SKILLS_DIR env var in gateway and CLI - Support optional runtime packages alongside dynamic skill loading - Update Dockerfile to COPY bundled-skills at /app/bundled-skills/ - Add PIP_CACHE_DIR in docker-entrypoint.sh for clean pip installs - Document bundled skills in 14-skills-runtime.md section 6 * feat(infra): remove ai-multimodal skill directory from bundled skills Remove the ai-multimodal skill package as part of consolidating runtime package support for bundled skills. This directory is no longer needed in the bundled skills structure. * feat(ci): add semantic release and Docker Hub publishing Add go-semantic-release workflow to auto-create semver tags on merge to main. Extend docker-publish to push all variants to both GHCR and Docker Hub (digitop/goclaw). * feat(skills): add system skills infrastructure with is_system column, dep scanning, and seeder - Migration 000017: add is_system boolean column with partial index - Store layer: UpsertSystemSkill, delete protection, IsSystemSkill - ListAccessible auto-includes system skills (no grants needed) - ListWithGrantStatus returns is_system field - Dependency scanner: auto-detect deps from scripts/ or skill-manifest.json - Dependency checker: verify system binaries, Python/Node packages - Seeder: seed bundled skills into DB on startup (idempotent via hash) - Gateway wiring: GOCLAW_BUNDLED_SKILLS_DIR env for bundled skills - HTTP: delete guard (403), slug conflict check (409), rescan-deps endpoint - UI: System badge, hide delete for system skills, rescan deps button - Agent skills tab: "Always available" for system skills - i18n: en/vi/zh keys for system skills, deps scanning * feat(skills): conditional system prompt, skill manifests, and Zip Slip fix - System prompt: only show package list when python3/node are available - Add skill-manifest.json for pdf, docx, xlsx, pptx bundled skills - Fix Zip Slip vulnerability in office/unpack.py (all 3 copies) * refactor(skills): extract shared office code to _shared/ and deduplicate Move office scripts (pack, unpack, validate, schemas, validators) from duplicated copies in docx/xlsx/pptx to skills/_shared/office/ with symlinks. Remove soffice.py (non-functional in containers) and update SKILL.md references to use soffice binary directly. Update seeder copyDir to follow symlinks. Removes ~45K lines of duplicate code across 3 skills. * fix(skills): address code review findings for system skills integration - H1: Remove dead symlink branch in copyDir (filepath.Walk follows symlinks) - H3: Fix rescan-deps to query ALL skills (including archived) and re-activate when deps become available; add ListAllSkills() + Status field to SkillInfo - H4: Add Status field to SkillCreateParams, stop overloading Visibility - M1: Batch Python/Node dep checks into single subprocess per runtime - M4: Add rows.Err() check in ListSkills to prevent caching partial results * feat(skills): async dep checking with realtime WS events Split Seed() into sync DB upsert + async CheckDepsAsync() goroutine. Gateway startup no longer blocks on Python/Node subprocess dep checks. - Seed() returns seeded skills list, all initially status="active" - CheckDepsAsync() runs in background, emits skill.deps.checked per-skill - skill.deps.complete event emitted when all checks finish - Each failed dep check: archives skill + BumpVersion() for immediate cache invalidation so next agent turn picks up the change - UI: use-query-invalidation listens to skill.deps.* events → auto-refresh skills list in realtime * feat(skills): system skills integration with toggle, dep checking, and per-item install - Add is_system, deps, enabled columns to skills table (migration 017) - Seed bundled core skills (pdf, docx, pptx, xlsx, skill-creator) on startup - PYTHONPATH-based dep detection — eliminates false positives from local modules - Per-item dep install UI with individual status (installing/success/error) - Enable/disable toggle for core and custom skills (independent of dep status) - Re-run dep check when skill is toggled back on - Inline skill thresholds: 40 skills / 5000 tokens before switching to search mode - Fix UpsertSystemSkill: backfill null file_hash without bumping DB version - Remove redundant skill-manifest.json files (replaced by deps JSONB column) - Show author from frontmatter in custom skills tab - Runtime checker for python3/pip3/node/npm availability - WS events for dep checking/installing progress - docs: add 15-core-skills-system.md, 16-skill-publishing.md --------- Co-authored-by: Goon <duy@wearetopgroup.com>
144 lines
4.4 KiB
Python
144 lines
4.4 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Skill Packager - Creates a distributable zip file of a skill folder
|
|
|
|
Usage:
|
|
python utils/package_skill.py <path/to/skill-folder> [output-directory]
|
|
|
|
Example:
|
|
python utils/package_skill.py skills/public/my-skill
|
|
python utils/package_skill.py skills/public/my-skill ./dist
|
|
"""
|
|
|
|
import fnmatch
|
|
import sys
|
|
import zipfile
|
|
from pathlib import Path
|
|
|
|
from encoding_utils import configure_utf8_console
|
|
from quick_validate import validate_skill
|
|
|
|
# Fix Windows console encoding for Unicode output (emojis, arrows)
|
|
configure_utf8_console()
|
|
|
|
# Exclusion patterns (from official Anthropic skill-creator)
|
|
EXCLUDE_DIRS = {'__pycache__', 'node_modules', '.git', '.DS_Store'}
|
|
EXCLUDE_GLOBS = {'*.pyc', '*.pyo', '.DS_Store', '*.egg-info'}
|
|
ROOT_EXCLUDE_DIRS = {'evals'} # Only excluded at skill root level
|
|
|
|
|
|
def package_skill(skill_path, output_dir=None):
|
|
"""
|
|
Package a skill folder into a zip file.
|
|
|
|
Args:
|
|
skill_path: Path to the skill folder
|
|
output_dir: Optional output directory for the zip file (defaults to current directory)
|
|
|
|
Returns:
|
|
Path to the created zip file, or None if error
|
|
"""
|
|
skill_path = Path(skill_path).resolve()
|
|
|
|
# Validate skill folder exists
|
|
if not skill_path.exists():
|
|
print(f"❌ Error: Skill folder not found: {skill_path}")
|
|
return None
|
|
|
|
if not skill_path.is_dir():
|
|
print(f"❌ Error: Path is not a directory: {skill_path}")
|
|
return None
|
|
|
|
# Validate SKILL.md exists
|
|
skill_md = skill_path / "SKILL.md"
|
|
if not skill_md.exists():
|
|
print(f"❌ Error: SKILL.md not found in {skill_path}")
|
|
return None
|
|
|
|
# Run validation before packaging
|
|
print("🔍 Validating skill...")
|
|
valid, message = validate_skill(skill_path)
|
|
if not valid:
|
|
print(f"❌ Validation failed: {message}")
|
|
print(" Please fix the validation errors before packaging.")
|
|
return None
|
|
print(f"✅ {message}\n")
|
|
|
|
# Determine output location
|
|
skill_name = skill_path.name
|
|
if output_dir:
|
|
output_path = Path(output_dir).resolve()
|
|
output_path.mkdir(parents=True, exist_ok=True)
|
|
else:
|
|
output_path = Path.cwd()
|
|
|
|
zip_filename = output_path / f"{skill_name}.zip"
|
|
|
|
# Create the zip file
|
|
try:
|
|
with zipfile.ZipFile(zip_filename, 'w', zipfile.ZIP_DEFLATED) as zipf:
|
|
# Walk through the skill directory, excluding unwanted files
|
|
skipped = []
|
|
for file_path in skill_path.rglob('*'):
|
|
if file_path.is_file():
|
|
rel = file_path.relative_to(skill_path)
|
|
parts = rel.parts
|
|
|
|
# Skip excluded directories
|
|
if any(p in EXCLUDE_DIRS for p in parts):
|
|
skipped.append(str(rel))
|
|
continue
|
|
|
|
# Skip root-only excluded dirs (e.g., evals/)
|
|
if parts[0] in ROOT_EXCLUDE_DIRS:
|
|
skipped.append(str(rel))
|
|
continue
|
|
|
|
# Skip excluded file patterns
|
|
if any(fnmatch.fnmatch(file_path.name, g) for g in EXCLUDE_GLOBS):
|
|
skipped.append(str(rel))
|
|
continue
|
|
|
|
arcname = file_path.relative_to(skill_path.parent)
|
|
zipf.write(file_path, arcname)
|
|
print(f" Added: {arcname}")
|
|
|
|
if skipped:
|
|
print(f"\n Skipped {len(skipped)} file(s): {', '.join(skipped[:5])}"
|
|
+ ("..." if len(skipped) > 5 else ""))
|
|
|
|
print(f"\n✅ Successfully packaged skill to: {zip_filename}")
|
|
return zip_filename
|
|
|
|
except Exception as e:
|
|
print(f"❌ Error creating zip file: {e}")
|
|
return None
|
|
|
|
|
|
def main():
|
|
if len(sys.argv) < 2:
|
|
print("Usage: python utils/package_skill.py <path/to/skill-folder> [output-directory]")
|
|
print("\nExample:")
|
|
print(" python utils/package_skill.py skills/public/my-skill")
|
|
print(" python utils/package_skill.py skills/public/my-skill ./dist")
|
|
sys.exit(1)
|
|
|
|
skill_path = sys.argv[1]
|
|
output_dir = sys.argv[2] if len(sys.argv) > 2 else None
|
|
|
|
print(f"📦 Packaging skill: {skill_path}")
|
|
if output_dir:
|
|
print(f" Output directory: {output_dir}")
|
|
print()
|
|
|
|
result = package_skill(skill_path, output_dir)
|
|
|
|
if result:
|
|
sys.exit(0)
|
|
else:
|
|
sys.exit(1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|