Module: PWN::AI::Agent::PolicyEvaluation

Defined in:
lib/pwn/ai/agent/policy_evaluation.rb

Overview

Opt-in, fixed local held-out evaluation. Never loaded by the online loop.

Constant Summary collapse

RUNNER = 
File.expand_path('../../../../scripts/benchmark_policy.rb', __dir__).freeze
MAX_SNAPSHOT_BYTES = 
4_194_304
WORKER_TIMEOUT_SECONDS = 
30

Class Method Summary collapse

Class Method Details

.authors ⇒ Object 



55
56
57
 
# File 'lib/pwn/ai/agent/policy_evaluation.rb', line 55

public_class_method def self.authors
  "AUTHOR(S):\n  0day Inc. <[email protected]>\n"
end

.evaluate(opts = {}) ⇒ Object

Raises:

  • (ArgumentError) 


20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
 
# File 'lib/pwn/ai/agent/policy_evaluation.rb', line 20

public_class_method def self.evaluate(opts = {})
  seed = opts.fetch(:seed, 0)
  raise ArgumentError, 'seed must be an integer in 0..7' unless seed.is_a?(Integer) && (0..7).cover?(seed)

  snapshots = i[baseline candidate].to_h { |arm| [arm, read_snapshot(path: opts.fetch(arm))] }
  Dir.mktmpdir('pwn-policy-evaluator-', '/tmp') do |root|
    Open3.popen3(
      { 'HOME' => root, 'TMPDIR' => root, 'LANG' => 'C.UTF-8' },
      RbConfig.ruby, RUNNER, '--snapshot-evaluation',
      unsetenv_others: true, chdir: root, pgroup: true
    ) do |input, output, error, child|
      readers = [output, error].map { |io| Thread.new { io.read } }
      begin
        Timeout.timeout(WORKER_TIMEOUT_SECONDS) do
          input.write(JSON.generate(snapshots: snapshots, seed: seed))
          input.close
          status = child.value
          stdout, stderr = readers.map(&:value)
          raise "held-out worker failed: #{stderr}" unless status.success?

          JSON.parse(stdout, symbolize_names: true)
        end
      ensure
        begin
          Process.kill('KILL', -child.pid)
        rescue Errno::ESRCH
          nil
        end
        child.join
        readers.each(&:join)
      end
    end
  end
end

.help ⇒ Object 



118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
 
# File 'lib/pwn/ai/agent/policy_evaluation.rb', line 118

public_class_method def self.help
  puts "USAGE:
    # Execute independently checked local tasks with frozen snapshots.
    #{self}.evaluate(
      baseline: 'required - explicit baseline policy JSON path',
      candidate: 'required - explicit candidate policy JSON path',
      seed: 'optional - held-out suite index, default 0'
    )

    # Explicitly install a candidate after fresh held-out replay; stop online writers first.
    #{self}.promote(
      enabled: 'optional - must be true to permit writes, default false',
      quiescent: 'required - true only after stopping all policy writers',
      baseline: 'required - explicit baseline policy JSON path',
      candidate: 'required - explicit candidate policy JSON path',
      reports: 'required - array of 2..8 independent evaluate reports',
      live_path: 'required - explicit existing target policy JSON path; no default'
    )

    # Restore the byte-exact pre-promotion snapshot, refusing intervening policy updates.
    #{self}.rollback(
      enabled: 'optional - must be true to permit writes, default false',
      quiescent: 'required - true only after stopping all policy writers',
      live_path: 'required - explicit existing target policy JSON path; no default',
      receipt: 'required - successful promote result with snapshot digests and paths'
    )

    # Display module authors.
    #{self}.authors
  "
  constants.sort
end

.promote(opts = {}) ⇒ Object

Explicit operator action only; never called from Policy.finish/Loop.



60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
 
# File 'lib/pwn/ai/agent/policy_evaluation.rb', line 60

public_class_method def self.promote(opts = {})
  return { promoted: false, reason: 'disabled' } unless opts[:enabled] == true
  raise ArgumentError, 'stop all policy writers and set quiescent: true' unless opts[:quiescent] == true

  baseline = read_snapshot(path: opts.fetch(:baseline))
  candidate = read_snapshot(path: opts.fetch(:candidate))
  live = File.expand_path(opts.fetch(:live_path))
  reports = opts.fetch(:reports)
  raise ArgumentError, 'need 2..8 distinct held-out reports' unless reports.is_a?(Array) && (2..8).cover?(reports.length) && reports.map { |r| r.fetch(:seed) }.uniq.length == reports.length
  raise ArgumentError, 'held-out reports must cover both filesystem environments' unless reports.map { |r| r.fetch(:seed).odd? }.uniq.length == 2

  replayed = reports.map do |report|
    fresh = evaluate(baseline: opts[:baseline], candidate: opts[:candidate], seed: report.fetch(:seed))
    raise ArgumentError, 'provenance/artifact replay mismatch' unless deterministic(value: report) == deterministic(value: fresh)

    gate(report: fresh)
    fresh
  end
  prior_digest = Digest::SHA256.hexdigest(baseline)
  candidate_digest = Digest::SHA256.hexdigest(candidate)
  raise ArgumentError, 'snapshot changed during evaluation' unless replayed.all? { |r| r[:snapshot_sha256] == { baseline: prior_digest, candidate: candidate_digest } }
  raise ArgumentError, 'live policy is not the evaluated baseline' unless read_snapshot(path: live) == baseline

  backup = "#{live}.rollback-#{prior_digest}.json"
  if File.exist?(backup)
    raise ArgumentError, 'rollback snapshot differs' unless read_snapshot(path: backup) == baseline
  else
    File.open(backup, File::WRONLY | File::CREAT | File::EXCL, 0o600) { |file| file.write(baseline) }
  end
  replace(path: live, bytes: candidate)
  { promoted: true, live_path: live, rollback_path: backup, baseline_sha256: prior_digest,
    candidate_sha256: candidate_digest, replayed_seeds: replayed.map { |r| r[:seed] } }
rescue StandardError => e
  { promoted: false, reason: e.message }
end

.rollback(opts = {}) ⇒ Object 



96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
 
# File 'lib/pwn/ai/agent/policy_evaluation.rb', line 96

public_class_method def self.rollback(opts = {})
  return { rolled_back: false, reason: 'disabled' } unless opts[:enabled] == true
  raise ArgumentError, 'stop all policy writers and set quiescent: true' unless opts[:quiescent] == true

  receipt = opts.fetch(:receipt)
  live = File.expand_path(opts.fetch(:live_path))
  prior_digest = receipt.fetch(:baseline_sha256)
  raise ArgumentError, 'invalid rollback digest' unless prior_digest.is_a?(String) && prior_digest.match?(/\A[0-9a-f]{64}\z/)

  backup = "#{live}.rollback-#{prior_digest}.json"
  raise ArgumentError, 'rollback target mismatch' unless receipt[:live_path] == live && receipt[:rollback_path] == backup

  bytes = read_snapshot(path: backup)
  raise ArgumentError, 'rollback snapshot digest mismatch' unless Digest::SHA256.hexdigest(bytes) == prior_digest
  raise ArgumentError, 'live policy changed since promotion' unless Digest::SHA256.hexdigest(read_snapshot(path: live)) == receipt.fetch(:candidate_sha256)

  replace(path: live, bytes: bytes)
  { rolled_back: true, live_path: live, restored_sha256: prior_digest }
rescue StandardError => e
  { rolled_back: false, reason: e.message }
end