{
  "ok": true,
  "resource": {
    "id": "RES_C4AF6EF9FB0D",
    "resource_type": "skill",
    "canonical_url": "https://github.com/wshobson/agents/tree/a30778f8c4e6b0a87567941b7cca4f534bf642b6/plugins/data-engineering/skills/spark-optimization",
    "name": "spark-optimization",
    "summary_en": "Optimize Apache Spark jobs with partitioning, caching, shuffle optimization, and memory tuning. Use when improving Spark performance, debugging slow jobs, or scaling data processing pipelines.",
    "summary_zh": "优化Apache Spark作业，包括分区、缓存、shuffle优化和内存调优。在提升Spark性能、调试慢作业或扩展数据处理管道时使用。",
    "description_en": "This Agent Skill provides production patterns for optimizing Apache Spark jobs, covering partitioning strategies, memory management, shuffle optimization, and performance tuning. The source document includes guidance on enabling Adaptive Query Execution (AQE), using columnar formats like Parquet/Delta, broadcasting small tables, monitoring the Spark UI, and right-sizing partitions. It also lists common pitfalls to avoid, such as collecting large data to the driver, unnecessary UDFs, over-caching, ignoring data skew, and using count() for existence checks. The skill is intended for use when improving Spark performance, debugging slow jobs, or scaling data processing pipelines.",
    "description_zh": "该Agent技能提供了优化Apache Spark作业的生产模式，涵盖分区策略、内存管理、shuffle优化和性能调优。源文档包括启用自适应查询执行（AQE）、使用Parquet/Delta等列式格式、广播小表、监控Spark UI以及合理设置分区大小的指导。还列出了应避免的常见陷阱，如将大数据收集到驱动程序、不必要的UDF、过度缓存、忽略数据倾斜以及使用count()检查存在性。该技能适用于提升Spark性能、调试慢作业或扩展数据处理管道。",
    "provider_name": "wshobson",
    "homepage_url": "https://github.com/wshobson/agents/tree/main/plugins/data-engineering/skills/spark-optimization",
    "endpoint_url": null,
    "source_url": "https://raw.githubusercontent.com/wshobson/agents/a30778f8c4e6b0a87567941b7cca4f534bf642b6/plugins/data-engineering/skills/spark-optimization/SKILL.md",
    "icon_url": null,
    "cover_image_url": null,
    "capabilities": [
      "data.optimize"
    ],
    "categories": [
      "data",
      "developer-tools"
    ],
    "industries": [
      "software",
      "general"
    ],
    "protocols": [
      "agent-skills"
    ],
    "auth": {
      "type": "none",
      "required": false
    },
    "pricing": {
      "model": "free"
    },
    "input_schema": {},
    "output_schema": {},
    "metadata": {
      "content_language": "en",
      "translation_status": "original-only",
      "license": {
        "spdx_id": "MIT",
        "source": "https://github.com/wshobson/agents/blob/a30778f8c4e6b0a87567941b7cca4f534bf642b6/LICENSE"
      },
      "github": {
        "repository": "wshobson/agents",
        "owner": "wshobson",
        "path": "plugins/data-engineering/skills/spark-optimization/SKILL.md",
        "commit": "a30778f8c4e6b0a87567941b7cca4f534bf642b6",
        "stars": 39366,
        "forks": 4194,
        "updated_at": "2026-09-03T02:19:37Z"
      },
      "skill": {
        "compatibility": null,
        "allowed_tools": null,
        "content_hash": "6af24df5e512113dac3238dcd9ed862c0a866ff91af92b95d024cf995959ac01",
        "line_count": 96
      },
      "review_gate": {
        "passed": true,
        "reasons": [],
        "policy_version": "review-gate-v1",
        "evaluated_at": "2026-09-03T06:01:59.589Z"
      }
    },
    "latest_version": "a30778f8c4e6",
    "status": "listed",
    "health_status": "unknown",
    "source_kind": "imported",
    "source_agent": "aiworkshub-skill-reviewer",
    "trust": {
      "signal": "none",
      "reason": "No suspicious or malicious evidence.",
      "signals": {
        "yellow": 0,
        "red": 0
      }
    },
    "review_summary": {
      "method_version": "capability-evidence-v2",
      "capability_definition": "partial",
      "contract_completeness": "natural_language",
      "access_friction": "low",
      "operational_transparency": "weak",
      "evidence_strength": "source_inspected",
      "use_readiness": "guided_use",
      "reviewed_at": "2026-09-03T06:02:00.405Z"
    },
    "experience_count": 0,
    "verified_experience_count": 0,
    "usage_count": 0,
    "success_rate": null,
    "first_seen_at": "2026-09-03T02:51:13.712Z",
    "last_seen_at": "2026-09-03T06:01:59.810Z",
    "published_at": "2026-09-03T06:01:59.810Z",
    "created_at": "2026-09-03T02:51:13.712Z",
    "updated_at": "2026-09-03T06:02:00.405Z",
    "api_url": "https://aiworkshub.io/api/v1/resources/RES_C4AF6EF9FB0D",
    "reviews_url": "https://aiworkshub.io/api/v1/resources/RES_C4AF6EF9FB0D/reviews",
    "public_url": "https://aiworkshub.io/resources/RES_C4AF6EF9FB0D",
    "distribution": {
      "canonical_identity": "https://github.com/wshobson/agents/tree/a30778f8c4e6b0a87567941b7cca4f534bf642b6/plugins/data-engineering/skills/spark-optimization",
      "syndicatable": true,
      "origins": [
        {
          "registry_url": "https://github.com/",
          "upstream_resource_id": "wshobson/agents:plugins/data-engineering/skills/spark-optimization/SKILL.md",
          "upstream_record_url": "https://github.com/wshobson/agents/tree/a30778f8c4e6b0a87567941b7cca4f534bf642b6/plugins/data-engineering/skills/spark-optimization",
          "relation": "syndicated",
          "metadata": {
            "source": "github-skill-import"
          },
          "first_seen_at": "2026-09-03T02:51:13.712Z",
          "last_seen_at": "2026-09-03T06:01:59.810Z"
        }
      ]
    },
    "tools": [],
    "review_profile": {
      "schema_version": "aiworkshub.resource-review/0.2",
      "method_version": "capability-evidence-v2",
      "review_state": {
        "identity": "verified",
        "source": "verified",
        "connectivity": "unknown",
        "contract": "weak",
        "safe_use": "untested",
        "real_use": "unobserved"
      },
      "assessment": {
        "capability_definition": "partial",
        "contract_completeness": "natural_language",
        "access_friction": "low",
        "operational_transparency": "weak",
        "evidence_strength": "source_inspected",
        "use_readiness": "guided_use"
      },
      "profile": {
        "capabilities": [
          {
            "id": "data.optimize",
            "weight": 100,
            "confidence": 80,
            "evidence_ids": [
              "source-record",
              "source-inspection"
            ]
          }
        ],
        "domains": [
          {
            "id": "data-engineering",
            "weight": 100,
            "confidence": 80,
            "evidence_ids": [
              "source-record",
              "source-inspection"
            ]
          }
        ],
        "workflow_roles": [
          "analyze",
          "transform"
        ]
      },
      "capability_units": [
        {
          "id": "skill:spark-optimization",
          "kind": "skill_action",
          "capability_id": "data.optimize",
          "effect": "unknown",
          "open_world": false,
          "idempotency": "unknown",
          "confirmation": "explicit policy",
          "input_contract": "natural_language",
          "output_contract": "natural_language",
          "accepts": [
            "task context"
          ],
          "produces": [
            "task result"
          ],
          "runtime_auth": {},
          "pricing": {},
          "evidence_ids": [
            "source-inspection"
          ],
          "metadata": {}
        }
      ],
      "access": {
        "distribution_license": {
          "spdx_id": "MIT",
          "source": "https://github.com/wshobson/agents/blob/a30778f8c4e6b0a87567941b7cca4f534bf642b6/LICENSE"
        },
        "runtime_auth": {},
        "runtime_pricing": {},
        "dependencies": []
      },
      "operational_flags": [],
      "trust": {
        "signal": "none",
        "evidence_ids": [],
        "reason": "No suspicious or malicious evidence."
      },
      "fit": {
        "good_for": [
          "Optimizing slow Spark jobs",
          "Tuning memory and executor configuration",
          "Implementing efficient partitioning strategies",
          "Debugging Spark performance issues",
          "Scaling Spark pipelines for large datasets",
          "Reducing shuffle and data skew"
        ],
        "not_for": [
          "Non-Spark data processing",
          "Real-time stream processing (not covered)"
        ]
      },
      "evidence": [
        {
          "id": "source-record",
          "type": "source_record",
          "source": "https://raw.githubusercontent.com/wshobson/agents/a30778f8c4e6b0a87567941b7cca4f534bf642b6/plugins/data-engineering/skills/spark-optimization/SKILL.md",
          "observation": "Submitted Skill record for spark-optimization; declared license MIT.",
          "observed_at": "2026-09-03T02:19:37Z",
          "payload": {}
        },
        {
          "id": "source-inspection",
          "type": "source_inspection",
          "source": "https://raw.githubusercontent.com/wshobson/agents/a30778f8c4e6b0a87567941b7cca4f534bf642b6/plugins/data-engineering/skills/spark-optimization/SKILL.md",
          "observation": "The Skill source was fetched and inspected (3160 bytes; SHA-256 6af24df5e512113dac3238dcd9ed862c0a866ff91af92b95d024cf995959ac01).",
          "observed_at": "2026-09-03T05:53:56.648Z",
          "payload": {}
        }
      ],
      "unknowns": [
        "Actual runtime behavior and effectiveness not observed.",
        "Compatibility with specific Spark versions or environments not specified.",
        "No information on required tools or dependencies beyond the skill content.",
        "No evidence of real-world usage or community adoption."
      ],
      "id": "REV_E31D6881A60F",
      "resource_id": "RES_C4AF6EF9FB0D",
      "evaluator_type": "platform_ai",
      "provider": "deepseek",
      "model": "deepseek-v4-flash",
      "review_scope": "source-inspected-skill",
      "created_at": "2026-09-03T06:02:00.405Z"
    }
  }
}