package sourcescan import ( "bytes" "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "io" "os" "path/filepath" ppconnector "knowledge-governance/internal/connectors/pp" ) const maxPPMetadataBytes int64 = 64 * 1024 * 1024 type FilePPScannerConfig struct { AllowedRoot string LedgerPath string ManifestPath string } type FilePPScanner struct { root string ledgerPath string manifestPath string } func NewFilePPScanner(config FilePPScannerConfig) (*FilePPScanner, error) { root, err := filepath.Abs(config.AllowedRoot) if err != nil || config.AllowedRoot == "" { return nil, errors.New("PP scan allowed root is required") } resolvedRoot, err := filepath.EvalSymlinks(root) if err != nil || resolvedRoot != root { return nil, errors.New("PP scan allowed root must be a canonical directory") } info, err := os.Lstat(root) if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { return nil, errors.New("PP scan allowed root must be a canonical directory") } if config.LedgerPath == "" || config.ManifestPath == "" { return nil, errors.New("PP ledger and manifest paths are required") } return &FilePPScanner{root: root, ledgerPath: config.LedgerPath, manifestPath: config.ManifestPath}, nil } func (s *FilePPScanner) Scan(ctx context.Context) (ppconnector.Snapshot, string, error) { if err := ctx.Err(); err != nil { return ppconnector.Snapshot{}, "", err } ledger, err := s.readFile(s.ledgerPath, maxPPMetadataBytes) if err != nil { return ppconnector.Snapshot{}, "", fmt.Errorf("read PP ledger: %w", err) } manifest, err := s.readFile(s.manifestPath, maxPPMetadataBytes) if err != nil { return ppconnector.Snapshot{}, "", fmt.Errorf("read PP manifest: %w", err) } if err := s.validateManifestPaths(manifest); err != nil { return ppconnector.Snapshot{}, "", err } snapshot, err := ppconnector.FoldLedger(bytes.NewReader(ledger)) if err != nil { return ppconnector.Snapshot{}, "", err } snapshot, _, err = ppconnector.RecoverMissingSHA256(snapshot, bytes.NewReader(manifest)) if err != nil { return ppconnector.Snapshot{}, "", err } snapshot, _, err = ppconnector.MergeOverLimitManifest(snapshot, bytes.NewReader(manifest)) if err != nil { return ppconnector.Snapshot{}, "", err } if err := ppconnector.ValidateSnapshot(snapshot); err != nil { return ppconnector.Snapshot{}, "", err } digest := sha256.New() _, _ = digest.Write(ledger) _, _ = digest.Write([]byte{0}) _, _ = digest.Write(manifest) return snapshot, hex.EncodeToString(digest.Sum(nil)), nil } func (s *FilePPScanner) validateManifestPaths(body []byte) error { var manifest struct { Results []struct { LocalPath string `json:"local_path"` Status string `json:"status"` } `json:"results"` } if err := json.Unmarshal(body, &manifest); err != nil { return fmt.Errorf("decode PP manifest paths: %w", err) } for _, item := range manifest.Results { if item.LocalPath == "" { return errors.New("PP manifest artifact local_path is required") } if item.Status == string(ppconnector.StatusOverLimit) { if err := s.validateDeclaredPath(item.LocalPath); err != nil { return fmt.Errorf("validate PP over-limit artifact path: %w", err) } file, err := s.openRegular(item.LocalPath) if errors.Is(err, os.ErrNotExist) { continue } if err != nil { return fmt.Errorf("validate PP over-limit artifact: %w", err) } if err := file.Close(); err != nil { return errors.New("close PP over-limit artifact") } continue } file, err := s.openRegular(item.LocalPath) if err != nil { return fmt.Errorf("validate PP manifest artifact: %w", err) } if err := file.Close(); err != nil { return errors.New("close PP manifest artifact") } } return nil } func (s *FilePPScanner) readFile(path string, maxBytes int64) ([]byte, error) { file, err := s.openRegular(path) if err != nil { return nil, err } body, readErr := io.ReadAll(io.LimitReader(file, maxBytes+1)) closeErr := file.Close() if readErr != nil { return nil, readErr } if closeErr != nil { return nil, closeErr } if int64(len(body)) > maxBytes { return nil, errors.New("PP metadata file exceeds size limit") } return body, nil } func (s *FilePPScanner) validateDeclaredPath(path string) error { absolute, err := filepath.Abs(path) if err != nil { return errors.New("invalid PP source path") } relative, err := filepath.Rel(s.root, absolute) if err != nil || relative == ".." || filepath.IsAbs(relative) || len(relative) >= 3 && relative[:3] == ".."+string(filepath.Separator) { return errors.New("PP source path is outside allowed root") } parent := filepath.Dir(absolute) resolvedParent, err := filepath.EvalSymlinks(parent) if err != nil || resolvedParent != parent { return errors.New("PP source path parent must not contain symlinks") } return nil } func (s *FilePPScanner) openRegular(path string) (*os.File, error) { if err := s.validateDeclaredPath(path); err != nil { return nil, err } absolute, _ := filepath.Abs(path) resolved, err := filepath.EvalSymlinks(absolute) if err != nil { return nil, err } if resolved != absolute { return nil, errors.New("PP source path must not contain symlinks") } before, err := os.Lstat(absolute) if err != nil { return nil, err } if !before.Mode().IsRegular() || before.Mode()&os.ModeSymlink != 0 { return nil, errors.New("PP source path must be a regular file") } file, err := os.Open(absolute) if err != nil { return nil, err } after, err := file.Stat() if err != nil || !after.Mode().IsRegular() || !os.SameFile(before, after) { _ = file.Close() return nil, errors.New("PP source file changed while opening") } return file, nil }