Module: PWN::AI::Agent::Policy

Defined in:
lib/pwn/ai/agent/policy.rb

Overview

PWN::AI::Agent::Policy is the LIVE tabular RL controller that pwn-ai did not have before R5. Everything else in the harness is retrieval-plus-policy: scores are written to disk and re-injected as prose, or exported later for optional LoRA. This module is the missing MDP:

state  s

Each Loop turn is one episode. Trusted environment/prerequisite bins scope fallback history; independently evidenced terminal attribution uses isolated action targets, not future-return credit for busywork. Transitions land in ~/.pwn/policy_traj.jsonl. Q(s,a) and REINFORCE logits H(s,a) are updated from those tuples and persisted in ~/.pwn/policy.json.

The learned Q values are an ADVISORY term in Registry.rank. They never replace TaskSummarizer planning, plan_first, or CORE_TOOLS. Disable with PWN::Env[:agent][:policy] = false.

Constant Summary collapse

POLICY_FILE = 
File.join(Dir.home, '.pwn', 'policy.json')
TRAJECTORY_FILE = 
File.join(Dir.home, '.pwn', 'policy_traj.jsonl')
ALPHA = 
0.15
ALPHA_PG = 
0.05
GAMMA = 
0.85
EPSILON = 
0.08
STEP_OK = 
0.0
STEP_FAIL = 
0.0
STEP_TASK = 
0.0
STEP_CLOSED = 
0.0
STEP_GRIND = 
0.0
STEP_COST = 
-0.01
STEP_COST_AFTER = 
8
MAX_TRAJ = 
2_000
GOLD_MIN = 
0.6
VISITS_MIN = 
2
CONTEXT_VISITS_MIN = 
3
COLD_EPISODES = 
8
WARM_EPISODES = 
40
TASK_MOD = 
16
ACTION_MOD = 
16
EP_KEY = 
:pwn_policy_episode
OPERATIONS = 
%w[read list search inspect write create update delete execute start stop status poll].freeze
RESULT_TYPES = 
%w[success failure timeout enoent eacces auth_required network syntax exit127 exit126 nonzero_exit handler_error invalid_payload].freeze
ENVIRONMENTS = 
%w[local remote container unknown].freeze
CAPABILITIES = 
%w[shell ruby python browser network filesystem credentials executable dependency service hardware].freeze
FAILURE_CATEGORIES = 
(%w[none unknown missing_prerequisite] + RESULT_TYPES).freeze
VERIFICATION_STATES = 
%w[unknown pending passed failed partial].freeze
ARGUMENT_ROLES = 
{
  'action' => 'operation', 'operation' => 'operation', 'op' => 'operation',
  'path' => 'path', 'file' => 'path', 'directory' => 'path',
  'command' => 'program', 'code' => 'program', 'query' => 'query',
  'url' => 'url', 'timeout' => 'control', 'limit' => 'control'
}.freeze

Class Method Summary collapse

Class Method Details

.advantage(opts = {}) ⇒ Object

Q(s,a) − V(s). Unknown / cold-start pairs return 0 so rank is unchanged.



468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
 
# File 'lib/pwn/ai/agent/policy.rb', line 468

public_class_method def self.advantage(opts = {})
  return 0.0 unless enabled?

  s = opts[:state] || current_state
  a = opts[:action].to_s
  return 0.0 if s.to_s.empty? || a.empty?

  tab = load
  visits = read_visit(table: tab, state: s, action: a)
  context = opts[:context_state]
  context = nil unless context.to_s.start_with?("#{s}~ctx:")
  contextual_visits = read_visit(table: tab, state: context, action: a)
  qsa = routing_q(table: tab, state: s, context_state: context, action: a)
  actions = ((tab[:q][s.to_s.to_sym] || {}).keys + (tab[:q][context.to_s.to_sym] || {}).keys).uniq
  baseline = actions.map { |act| routing_q(table: tab, state: s, context_state: context, action: act) }.max || 0.0
  # Unknown candidates must not tie an established successful action.
  return 0.0 if contextual_visits < CONTEXT_VISITS_MIN && visits < VISITS_MIN && [qsa.abs, baseline.abs].max < 0.08

  (qsa - baseline).clamp(-1.0, 1.0).round(4)
rescue StandardError
  0.0
end

.attach_episode!(opts = {}) ⇒ Object 



548
549
550
551
 
# File 'lib/pwn/ai/agent/policy.rb', line 548

public_class_method def self.attach_episode!(opts = {})
  Thread.current[EP_KEY] = opts[:episode]
  opts[:episode]
end

.authors ⇒ Object 



750
751
752
 
# File 'lib/pwn/ai/agent/policy.rb', line 750

public_class_method def self.authors
  "AUTHOR(S):\n  0day Inc. <[email protected]>\n"
end

.begin_episode(opts = {}) ⇒ Object

Supported Method Parameters

ep = PWN::AI::Agent::Policy.begin_episode( session_id: 'optional - PWN::Sessions id', request: 'optional - user request', kind: 'optional - request kind', intent: 'optional - Loop.request_intent', engine: 'optional - active engine', ts_state: 'optional - TaskSummarizer state hash' )



180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
 
# File 'lib/pwn/ai/agent/policy.rb', line 180

public_class_method def self.begin_episode(opts = {})
  return { skipped: :disabled } unless enabled?

  sid = (opts[:session_id] || "ep_#{Thread.current.object_id}").to_s
  task = active_task_text(ts_state: opts[:ts_state], request: opts[:request])
  s0 = state(
    kind: opts[:kind],
    request: task,
    last_action: 'start',
    fails: 0,
    engine: opts[:engine],
    ts_state: opts[:ts_state],
    trusted_context: opts[:trusted_context]
  )
  ep = {
    session_id: sid,
    request: opts[:request].to_s.dup,
    kind: normalize_kind(raw: opts[:kind]),
    intent: opts[:intent].to_s,
    engine: opts[:engine].to_s,
    trusted_context: (observed_context(trusted_context: opts[:trusted_context]) if opts[:trusted_context].is_a?(Hash)),
    started_at: Time.now.utc.iso8601,
    state: s0,
    last_action: 'start',
    plan_idx: ts_idx(ts_state: opts[:ts_state]),
    plan_open: ts_open?(ts_state: opts[:ts_state]),
    fails: 0,
    action_ids: {},
    steps: []
  }
  Thread.current[EP_KEY] = ep
  ep
rescue StandardError => e
  warn "[pwn-ai/policy] begin_episode swallowed: #{e.class}: #{e.message}"
  nil
end

.cold? ⇒ Boolean 



135
136
137
138
139
 
# File 'lib/pwn/ai/agent/policy.rb', line 135

public_class_method def self.cold?
  stats[:n_episodes].to_i < COLD_EPISODES
rescue StandardError
  true
end

.current_context_state ⇒ Object

Previous action features are available BEFORE the next tool choice; the candidate's own result must never leak into its decision state.



533
534
535
536
 
# File 'lib/pwn/ai/agent/policy.rb', line 533

public_class_method def self.current_context_state
  ep = current_episode
  ep.is_a?(Hash) ? ep[:context_state] : nil
end

.current_episode ⇒ Object 



527
528
529
 
# File 'lib/pwn/ai/agent/policy.rb', line 527

public_class_method def self.current_episode
  Thread.current[EP_KEY]
end

.current_state ⇒ Object 



522
523
524
525
 
# File 'lib/pwn/ai/agent/policy.rb', line 522

public_class_method def self.current_state
  ep = Thread.current[EP_KEY]
  ep.is_a?(Hash) ? ep[:state] : nil
end

.detach_episode! ⇒ Object

Hermes split: snapshot + clear the live episode so Loop.maybe_finish_policy is a no-op on the user-visible path while TurnFinalizer re-attaches it on the background review thread.



542
543
544
545
546
 
# File 'lib/pwn/ai/agent/policy.rb', line 542

public_class_method def self.detach_episode!
  ep = Thread.current[EP_KEY]
  Thread.current[EP_KEY] = nil
  ep
end

.enabled? ⇒ Boolean 



730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
 
# File 'lib/pwn/ai/agent/policy.rb', line 730

public_class_method def self.enabled?
  return false unless defined?(PWN::Env) && PWN::Env.is_a?(Hash)

  v = begin
    PWN::Env.dig(:ai, :agent, :policy)
  rescue StandardError
    nil
  end
  return false if v == false

  if defined?(Metrics) && Metrics.respond_to?(:calibration)
    cal = Metrics.calibration
    return false if cal[:n].to_i >= 8 && !Metrics.calibration_green?
  end

  true
rescue StandardError
  false
end

.episode_budget_met? ⇒ Boolean

True once live returns, warmup-replayed trajectories, or a warmed Q table have enough mass to emit greedy suggestions. Cold? stays a coarser state-encoding gate; this is the banner.



150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
 
# File 'lib/pwn/ai/agent/policy.rb', line 150

public_class_method def self.episode_budget_met?
  n = stats[:n_episodes].to_i
  return true if n >= COLD_EPISODES

  tab = load
  warmed = !tab[:warmed_at].to_s.empty?
  pairs = 0
  tab[:q].each_value { |acts| pairs += acts.length if acts.is_a?(Hash) }
  return true if warmed && pairs >= COLD_EPISODES

  traj_n = trajectories(limit: COLD_EPISODES).count { |ep| !ep[:score].nil? }
  warmed && traj_n >= COLD_EPISODES
rescue StandardError
  false
end

.evaluate(opts = {}) ⇒ Object

Replay stored trajectories under the current Q table. Does not write. Used by task 7 (evaluate policy quality).



627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
 
# File 'lib/pwn/ai/agent/policy.rb', line 627

public_class_method def self.evaluate(opts = {})
  rows = trajectories(limit: opts[:limit] || 200).reject { |ep| ep[:score].nil? }
  return { n: 0, mean_return: nil, greedy_match: nil, mean_abs_td: nil } if rows.empty?

  tab = load
  abs_td = []
  greedy_hits = 0
  greedy_n = 0
  rows.each do |ep|
    Array(ep[:steps]).flat_map { |tr| transition_variants(transition: tr) }.each do |tr|
      s = tr[:state].to_s
      a = tr[:action].to_s
      next if s.empty? || a.empty?

      qsa = read_q(table: tab, state: s, action: a)
      max_n = tr[:terminal] ? 0.0 : max_q(table: tab, state: tr[:next_state])
      abs_td << (tr[:reward].to_f + (GAMMA * max_n) - qsa).abs
      acts = (tab[:q][s.to_sym] || {}).keys.map(&:to_s)
      next if acts.empty?

      greedy_n += 1
      best = acts.max_by { |act| read_q(table: tab, state: s, action: act) }
      greedy_hits += 1 if best == a
    end
  end
  rets = rows.map { |r| r[:return].to_f }
  {
    n: rows.length,
    mean_return: (rets.sum / rets.length).round(4),
    mean_abs_td: abs_td.empty? ? nil : (abs_td.sum / abs_td.length).round(4),
    greedy_match: greedy_n.positive? ? (greedy_hits.to_f / greedy_n).round(3) : nil
  }
rescue StandardError => e
  { n: 0, error: "#{e.class}: #{e.message}" }
end

.finish(opts = {}) ⇒ Object

Supported Method Parameters

report = PWN::AI::Agent::Policy.finish( session_id: 'optional - active episode id', score: 'optional - Reward.judge 0..1 (training target)', attribution: 'optional - trusted independent_verifier|controlled_comparison, verified_action_ids: []', verdict: 'optional - solved|partial|wrong|refused', proxy_ok: 'optional - Boolean fallback when no judge score' )



315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
 
# File 'lib/pwn/ai/agent/policy.rb', line 315

public_class_method def self.finish(opts = {})
  return { skipped: :disabled } unless enabled?

  ep = Thread.current[EP_KEY]
  return { skipped: :no_episode } unless ep.is_a?(Hash)

  return { skipped: :session_mismatch } if opts[:session_id] && ep[:session_id] && opts[:session_id].to_s != ep[:session_id].to_s

  unless ep[:steps].empty?
    last = ep[:steps].last
    last[:next_state] = state(
      kind: ep[:kind],
      request: ep[:request],
      last_action: last[:action] || ep[:last_action],
      fails: ep[:fails],
      engine: ep[:engine],
      ts_state: opts[:ts_state],
      final: opts[:final],
      score: opts[:score],
      trusted_context: ep[:trusted_context]
    )
  end
  terminal = terminal_reward(
    score: opts[:score],
    proxy_ok: opts[:proxy_ok],
    confidence: opts[:confidence]
  )
  attribute_terminal!(episode: ep, reward: terminal, attribution: opts[:attribution])
  ep[:steps].last[:terminal] = true
  ep[:score] = opts[:score]
  ep[:verdict] = opts[:verdict]
  ep[:return] = opts[:score].nil? ? nil : discounted_return(steps: ep[:steps])
  ep[:ended_at] = Time.now.utc.iso8601

  n_td = 0
  n_pg = 0
  ep[:steps].each_with_index do |tr, idx|
    next if opts[:score].nil?

    g = tr[:credit_mode] ? tr[:reward].to_f : discounted_return(steps: ep[:steps][idx..])
    transition_variants(transition: tr).each do |variant|
      n_td += 1 if update_q!(transition: variant)
      advantage = g - value(state: variant[:state])
      advantage = [advantage, 0.0].min unless g.positive?
      n_pg += 1 if update_pg!(state: variant[:state], action: variant[:action], advantage: advantage)
    end
  end

  persist_episode!(episode: ep)
  Thread.current[EP_KEY] = nil
  {
    session_id: ep[:session_id],
    steps: ep[:steps].length,
    return: ep[:return],
    score: opts[:score],
    terminal_reward: terminal,
    attribution: ep[:attribution],
    td_updates: n_td,
    pg_updates: n_pg
  }
rescue StandardError => e
  warn "[pwn-ai/policy] finish swallowed: #{e.class}: #{e.message}"
  Thread.current[EP_KEY] = nil
  { error: "#{e.class}: #{e.message}" }
end

.help ⇒ Object 



754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
 
# File 'lib/pwn/ai/agent/policy.rb', line 754

public_class_method def self.help
  puts "USAGE:
    # Feature → discrete state
    #{self}.state(
      kind: 'optional - statement|question|autonomous_goal|…',
      request: 'optional - user text / active English task',
      last_action: 'optional - previous tool name',
      fails: 'optional - in-turn failure count',
      engine: 'optional - active engine',
      request_kind: 'optional - request kind value consumed by #state',
      task: 'optional - task value consumed by #state',
      ts_state: 'optional - ts state value consumed by #state',
      final: 'optional - final value consumed by #state',
      score: 'optional - score value consumed by #state',
      last: 'optional - last value consumed by #state',
      fail_n: 'optional - fail n value consumed by #state',
      trusted_context: 'optional - trusted environment, capability booleans, missing prerequisite categories, failure category and verification state'
    )

    # Sanitize caller-only observations to fixed allowlisted categories.
    #{self}.observed_context(
      trusted_context: 'optional - environment: local|remote|container|unknown; capabilities: Hash of booleans; missing_prerequisites: capability names; failure_category; verification_state: unknown|pending|passed|failed|partial'
    )

    # Scope a live state without discarding task/engine/plan bins.
    #{self}.observed_state(
      state: 'required - existing decision state',
      trusted_context: 'optional - observations accepted by observed_context'
    )

    # Run cold and return its result
    #{self}.cold?

    # Run warm and return its result
    #{self}.warm?

    # True once live returns, warmup-replayed trajectories, or a
    #{self}.episode_budget_met?

    # Episode / environment loop
    #{self}.begin_episode(
      session_id: 'optional - PWN::Sessions id (defaults to ep_)',
      request: 'optional - user request',
      kind: 'optional - request kind',
      intent: 'optional - Loop.request_intent',
      engine: 'optional - active engine',
      ts_state: 'optional - TaskSummarizer state hash',
      trusted_context: 'optional - trusted observations accepted by observed_context'
    )

    # Run observe step and return its result
    #{self}.observe_step(
      session_id: 'optional - must match begin_episode when set',
      action: 'required - tool name',
      action_id: 'optional - trusted dispatch ID, stored only as an episode-local ordinal',
      trusted_context: 'optional - observations after this step; partial updates retain earlier categories',
      args: 'optional - arguments reduced to fixed roles/types only',
      operation: 'optional - allowlisted operation override',
      result_type: 'optional - allowlisted semantic result shape',
      ok: 'required - Boolean, Reward.semantic_ok',
      duration: 'optional - Float seconds',
      ts_state: 'optional - TaskSummarizer state',
      request: 'optional - used if episode was not begun',
      kind: 'optional - kind value consumed by #observe_step',
      engine: 'optional - engine value consumed by #observe_step'
    )

    # Run finish and return its result
    #{self}.finish(
      session_id: 'optional - active episode id',
      score: 'optional - Reward.judge 0..1 (training target)',
      verdict: 'optional - solved|partial|wrong|refused',
      proxy_ok: 'optional - Boolean fallback when no judge score',
      ts_state: 'optional - ts state value consumed by #finish',
      final: 'optional - final value consumed by #finish',
      confidence: 'optional - confidence value consumed by #finish',
      attribution: 'optional - caller-only {source: independent_verifier|controlled_comparison, verified_action_ids: dispatch IDs}; missing or ambiguous IDs earn no tool credit; nil score never trains'
    )

    # Value-based update (Q-learning) + policy-gradient (REINFORCE)
    #{self}.update_q!(
      transition: 'required - Hash with :state :action :reward :next_state :terminal (defaults to opts)'
    )

    # Run update pg and return its result
    #{self}.update_pg!(
      state: 'required - state value consumed by #update_pg!',
      action: 'required - action value consumed by #update_pg!',
      advantage: 'required - scalar G_t − V(s)'
    )

    # Run q and return its result
    #{self}.q(
      state: 'optional - state value consumed by #q',
      action: 'optional - action value consumed by #q'
    )

    # Run value and return its result
    #{self}.value(
      state: 'optional - state value consumed by #value'
    )

    # Q(s,a) − V(s). Unknown / cold-start pairs return 0 so rank is unchanged
    #{self}.advantage(
      state: 'optional - state value consumed by #advantage (defaults to current_state)',
      action: 'optional - action value consumed by #advantage',
      context_state: 'optional - previous action context supplied by Registry'
    )

    # Run recommend and return its result
    #{self}.recommend(
      state: 'optional - default current episode state',
      actions: 'required - Array of tool names',
      context_state: 'optional - previous action context (default live context)',
      trusted_context: 'optional - observed prerequisites used to exclude unavailable actions',
      epsilon: 'optional - explore probability (default EPSILON)'
    )

    # Run current state and return its result
    #{self}.current_state

    # Run current episode and return its result
    #{self}.current_episode

    # Sanitized previous-action state used for advisory routing
    #{self}.current_context_state

    # Hermes split: snapshot + clear the live episode so Loop.maybe_finish_policy
    #{self}.detach_episode!

    # Run attach episode and return its result
    #{self}.attach_episode!(
      episode: 'optional - episode value consumed by #attach_episode!'
    )

    # Persistence / eval
    #{self}.load

    # Run save and return its result
    #{self}.save(
      table: 'optional - table value consumed by #save (defaults to load)'
    )

    # Run trajectories and return its result
    #{self}.trajectories(
      limit: 'optional - limit value consumed by #trajectories'
    )

    # Run stats and return its result
    #{self}.stats

    # Replay stored trajectories under the current Q table
    #{self}.evaluate(
      limit: 'optional - limit value consumed by #evaluate'
    )

    # Run to context and return its result
    #{self}.to_context(
      limit: 'optional - limit value consumed by #to_context'
    )

    # Run lean and return its result
    #{self}.lean!(
      dry_run: 'optional - dry run value consumed by #lean!'
    )

    # Run reset and return its result
    #{self}.reset

    # Run enabled and return its result
    #{self}.enabled?

    # Print the AUTHOR(S) string for this module.
    #{self}.authors

    # Replay stored trajectories into Q so a cold table is not empty advice
    #{self}.warmup!(
      limit: 'optional - limit value consumed by #warmup!'
    )

    # Run maybe warmup and return its result
    #{self}.maybe_warmup!
  "
  constants.sort
end

.lean!(opts = {}) ⇒ Object 



697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
 
# File 'lib/pwn/ai/agent/policy.rb', line 697

public_class_method def self.lean!(opts = {})
  dry = opts[:dry_run] ? true : false
  return { skipped: true } unless File.exist?(TRAJECTORY_FILE)

  rows = File.readlines(TRAJECTORY_FILE)
  keep = []
  rows.each do |line|
    ep = JSON.parse(line, symbolize_names: true)
    gold = ep[:return].to_f >= GOLD_MIN || ep[:score].to_f >= GOLD_MIN
    keep << [ep, line, gold]
  rescue StandardError
    next
  end
  gold = keep.select { |_, _, g| g }.map { |_, line, _| line }
  rest = keep.reject { |_, _, g| g }.last([MAX_TRAJ - gold.length, 0].max).map { |_, line, _| line }
  out = gold + rest
  unless dry
    tmp = "#{TRAJECTORY_FILE}.#{Process.pid}.tmp"
    File.write(tmp, out.join)
    File.rename(tmp, TRAJECTORY_FILE)
  end
  { removed: rows.length - out.length, remaining: out.length, dry_run: dry }
rescue StandardError => e
  { error: "#{e.class}: #{e.message}" }
end

.load ⇒ Object

Persistence / eval



557
558
559
560
561
562
563
564
565
566
567
568
569
 
# File 'lib/pwn/ai/agent/policy.rb', line 557

public_class_method def self.load
  FileUtils.mkdir_p(File.dirname(POLICY_FILE))
  return blank_table unless File.exist?(POLICY_FILE)

  data = JSON.parse(File.read(POLICY_FILE), symbolize_names: true)
  data[:q] = {} unless data[:q].is_a?(Hash)
  data[:h] = {} unless data[:h].is_a?(Hash)
  data[:visits] = {} unless data[:visits].is_a?(Hash)
  data[:returns] = Array(data[:returns])
  data
rescue StandardError
  blank_table
end

.maybe_warmup! ⇒ Object 



1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
 
# File 'lib/pwn/ai/agent/policy.rb', line 1373

public_class_method def self.maybe_warmup!
  return { skipped: :disabled } unless enabled?
  return { skipped: :no_traj } unless File.exist?(TRAJECTORY_FILE)

  tab = load
  pairs = 0
  tab[:q].each_value { |acts| pairs += acts.length if acts.is_a?(Hash) }
  return { skipped: :warmed } if !tab[:warmed_at].to_s.empty? && pairs >= COLD_EPISODES && !cold?

  warmup!
rescue StandardError
  { skipped: :error }
end

.observe_step(opts = {}) ⇒ Object

Supported Method Parameters

step = PWN::AI::Agent::Policy.observe_step( session_id: 'optional - must match begin_episode when set', action: 'required - tool name', action_id: 'optional - trusted dispatch ID; mapped to a persisted episode-local ID', trusted_context: 'optional - caller observations after this step; never model claims', args: 'optional - arguments, reduced to fixed roles/types, never stored raw', operation: 'optional - known operation override; otherwise derived from args', result_type: 'optional - Reward.semantic_ok shape; fixed allowlist only', ok: 'required - Boolean, Reward.semantic_ok', duration: 'optional - Float seconds', ts_state: 'optional - TaskSummarizer state', request: 'optional - used if episode was not begun', kind: 'optional', engine: 'optional' )



234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
 
# File 'lib/pwn/ai/agent/policy.rb', line 234

public_class_method def self.observe_step(opts = {})
  return { skipped: :disabled } unless enabled?

  action = opts[:action].to_s
  return { skipped: :no_action } if action.empty?

  ep = Thread.current[EP_KEY]
  if ep.nil? || (opts[:session_id] && ep[:session_id] && opts[:session_id].to_s != ep[:session_id].to_s)
    begin_episode(
      session_id: opts[:session_id],
      request: opts[:request],
      kind: opts[:kind],
      engine: opts[:engine],
      ts_state: opts[:ts_state],
      trusted_context: opts[:trusted_context]
    )
    ep = Thread.current[EP_KEY]
  end
  return { skipped: :no_episode } unless ep.is_a?(Hash)

  ok = opts[:ok] ? true : false
  ep[:fails] = ep[:fails].to_i + 1 unless ok
  reward = 0.0
  reward = STEP_COST if ep[:steps].length >= STEP_COST_AFTER
  ep[:plan_idx] = ts_idx(ts_state: opts[:ts_state])
  ep[:plan_open] = ts_open?(ts_state: opts[:ts_state])
  s = ep[:state]
  task = active_task_text(ts_state: opts[:ts_state], request: ep[:request])
  if opts[:trusted_context].is_a?(Hash)
    previous = ep[:trusted_context] || {}
    observed = previous.merge(opts[:trusted_context])
    observed[:capabilities] = (previous[:capabilities] || {}).merge(opts[:trusted_context][:capabilities] || {})
    ep[:trusted_context] = observed_context(trusted_context: observed)
  end
  s2 = state(
    kind: ep[:kind],
    request: task,
    last_action: action,
    fails: ep[:fails],
    engine: ep[:engine],
    ts_state: opts[:ts_state],
    trusted_context: ep[:trusted_context]
  )
  context = action_context(opts)
  context_s2 = contextual_state(state: s2, action: action, context: context)
  trans = {
    state: s,
    action: action,
    action_id: "a#{ep[:steps].length + 1}",
    action_context: context,
    context_state: ep[:context_state],
    next_context_state: context_s2,
    reward: reward,
    next_state: s2,
    ok: ok,
    duration: opts[:duration].to_f,
    terminal: false
  }
  external_id = (opts[:action_id] || trans[:action_id]).to_s
  ep[:action_ids] ||= {}
  ep[:action_ids][external_id] ||= []
  ep[:action_ids][external_id] << trans[:action_id]
  ep[:steps] << trans
  ep[:state] = s2
  ep[:context_state] = context_s2
  ep[:last_action] = action
  trans
rescue StandardError => e
  warn "[pwn-ai/policy] observe_step swallowed: #{e.class}: #{e.message}"
  nil
end

.observed_context(opts = {}) ⇒ Object

Caller-only observations, never model claims or raw probe output.



118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
 
# File 'lib/pwn/ai/agent/policy.rb', line 118

public_class_method def self.observed_context(opts = {})
  raw = opts[:trusted_context].is_a?(Hash) ? opts[:trusted_context] : {}
  environment = raw[:environment].to_s
  environment = 'unknown' unless ENVIRONMENTS.include?(environment)
  capabilities = raw[:capabilities].is_a?(Hash) ? raw[:capabilities] : {}
  capabilities = CAPABILITIES.each_with_object({}) do |name, result|
    value = capabilities.key?(name.to_sym) ? capabilities[name.to_sym] : capabilities[name]
    result[name.to_sym] = value if [true, false].include?(value)
  end
  missing = Array(raw[:missing_prerequisites]).map(&:to_s).intersection(CAPABILITIES).sort
  failure = raw[:failure_category].to_s
  failure = 'unknown' unless FAILURE_CATEGORIES.include?(failure)
  verification = raw[:verification_state].to_s
  verification = 'unknown' unless VERIFICATION_STATES.include?(verification)
  { environment: environment, capabilities: capabilities, missing_prerequisites: missing, failure_category: failure, verification_state: verification }
end

.observed_state(opts = {}) ⇒ Object

Add observations without losing the live engine/task/plan bins.



106
107
108
109
110
111
112
113
114
115
 
# File 'lib/pwn/ai/agent/policy.rb', line 106

public_class_method def self.observed_state(opts = {})
  return opts[:state] unless opts[:trusted_context].is_a?(Hash)

  parts = opts[:state].to_s.split('|')
  engine = parts.pop
  parts.reject! { |part| part.start_with?('env:', 'obs:') }
  observed = observed_context(trusted_context: opts[:trusted_context])
  environment = observed.slice(:environment, :capabilities, :missing_prerequisites)
  "#{parts.join('|')}|env:#{JSON.generate(environment)}|obs:#{observed[:failure_category]}:#{observed[:verification_state]}|#{engine}"
end

.q(opts = {}) ⇒ Object

Query — used by Registry.rank (advisory only)



454
455
456
457
458
 
# File 'lib/pwn/ai/agent/policy.rb', line 454

public_class_method def self.q(opts = {})
  read_q(table: load, state: opts[:state], action: opts[:action])
rescue StandardError
  0.0
end

.recommend(opts = {}) ⇒ Object

Supported Method Parameters

pick = PWN::AI::Agent::Policy.recommend( state: 'optional - default current episode state', actions: 'required - Array of tool names', epsilon: 'optional - explore probability (default EPSILON)' )



498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
 
# File 'lib/pwn/ai/agent/policy.rb', line 498

public_class_method def self.recommend(opts = {})
  actions = Array(opts[:actions]).map(&:to_s).reject(&:empty?)
  actions = actions.select { |action| Registry.available?(name: action, trusted_context: opts[:trusted_context]) } if defined?(Registry)
  return { action: nil, reason: :empty } if actions.empty?

  s = opts[:state] || current_state || 'unknown'
  eps = if opts.key?(:epsilon)
          opts[:epsilon].to_f
        else
          oc = 0.0
          oc = Metrics.calibration[:overconfidence].to_f if defined?(Metrics) && Metrics.respond_to?(:calibration)
          (EPSILON + [oc, 0.0].max).clamp(EPSILON, 0.45)
        end
  return { action: actions.sample, reason: :explore, state: s, epsilon: eps } if rand < eps

  tab = load
  context = opts.key?(:context_state) ? opts[:context_state] : (current_context_state if s == current_state)
  scored = actions.map { |a| [a, routing_q(table: tab, state: s, context_state: context, action: a)] }
  best = scored.max_by { |_, v| v }
  { action: best[0], q: best[1].round(4), reason: :greedy, state: s, ranked: scored.sort_by { |_, v| -v } }
rescue StandardError => e
  { action: Array(opts[:actions]).first, reason: :error, error: e.message }
end

.reset ⇒ Object 



723
724
725
726
727
728
 
# File 'lib/pwn/ai/agent/policy.rb', line 723

public_class_method def self.reset
  FileUtils.rm_f(POLICY_FILE)
  FileUtils.rm_f(TRAJECTORY_FILE)
  Thread.current[EP_KEY] = nil
  blank_table
end

.save(opts = {}) ⇒ Object 



571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
 
# File 'lib/pwn/ai/agent/policy.rb', line 571

public_class_method def self.save(opts = {})
  table = opts[:table] || load
  table[:updated_at] = Time.now.utc.iso8601
  FileUtils.mkdir_p(File.dirname(POLICY_FILE))
  path = POLICY_FILE
  tmp = File.join(File.dirname(path), ".#{File.basename(path)}.#{Process.pid}.tmp")
  File.open(tmp, File::WRONLY | File::CREAT | File::TRUNC, 0o644) do |f|
    f.flock(File::LOCK_EX)
    f.write(JSON.pretty_generate(table))
    f.flush
    f.fsync
  end
  File.rename(tmp, path)
  table
ensure
  FileUtils.rm_f(tmp) if defined?(tmp) && tmp && File.exist?(tmp)
end

.state(opts = {}) ⇒ Object

Supported Method Parameters

key = PWN::AI::Agent::Policy.state( kind: 'optional - statement|question|autonomous_goal|…', request: 'optional - user text / active English task', last_action: 'optional - previous tool name', fails: 'optional - in-turn failure count', engine: 'optional - active engine' )



82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
 
# File 'lib/pwn/ai/agent/policy.rb', line 82

public_class_method def self.state(opts = {})
  kind = normalize_kind(raw: opts[:kind] || opts[:request_kind])
  task = task_family(text: opts[:task] || opts[:request])
  eng  = opts[:engine].to_s.empty? ? 'any' : opts[:engine].to_s.downcase
  plan_q = plan_quality_bin(ts_state: opts[:ts_state])
  comp = completeness_bin(final: opts[:final], score: opts[:score])
  use = usable_bin(final: opts[:final], score: opts[:score])
  # Three independent bins so Q can see plan quality, answer
  # completeness, and whether the human actually got a usable result.
  qual = "p#{plan_q}c#{comp}u#{use}"
  key = if warm?
          last = action_bucket(name: opts[:last_action] || opts[:last] || 'start')
          fail = fail_bin(count: opts[:fails] || opts[:fail_n])
          "#{kind}|#{task}|a#{last}|f#{fail}|#{qual}|#{eng}"
        elsif !cold?
          fail = fail_bin(count: opts[:fails] || opts[:fail_n])
          "#{kind}|#{task}|f#{fail}|#{qual}|#{eng}"
        else
          "#{kind}|#{task}|#{qual}|#{eng}"
        end
  observed_state(state: key, trusted_context: opts[:trusted_context])
end

.stats ⇒ Object 



602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
 
# File 'lib/pwn/ai/agent/policy.rb', line 602

public_class_method def self.stats
  tab = load
  q_pairs = 0
  tab[:q].each_value { |acts| q_pairs += acts.length if acts.is_a?(Hash) }
  rets = Array(tab[:returns])
  mean_r = rets.empty? ? nil : (rets.sum.to_f / rets.length).round(4)
  n_up = tab[:n_updates].to_i
  td_mean = n_up.positive? ? (tab[:td_abs_sum].to_f / n_up).round(4) : nil
  {
    enabled: enabled?,
    n_updates: n_up,
    n_states: tab[:q].length,
    n_pairs: q_pairs,
    n_episodes: rets.length,
    mean_return: mean_r,
    mean_abs_td: td_mean,
    alpha: ALPHA,
    gamma: GAMMA,
    epsilon: EPSILON
  }
end

.to_context(opts = {}) ⇒ Object 



663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
 
# File 'lib/pwn/ai/agent/policy.rb', line 663

public_class_method def self.to_context(opts = {})
  return '' unless enabled?

  maybe_warmup! unless episode_budget_met?
  s = stats
  lines = []
  unless episode_budget_met?
    lines << 'POLICY (R5 tabular Q / REINFORCE — advisory only, does not replace planning)'
    lines << "  policy cold episodes=#{s[:n_episodes]}/#{COLD_EPISODES} — omit greedy suggestion"
    return "#{lines.join("\n")}\n"
  end

  ev = evaluate(limit: opts[:limit] || 40)
  fallback = PWN::AI::Agent::Registry::DEFAULT_PREFERENCE
  pref = begin
    PWN::AI::Agent::Registry.preference_order
  rescue StandardError
    []
  end
  actions = pref.empty? ? fallback : pref
  rec = begin
    recommend(actions: actions, epsilon: 0.0)[:action]
  rescue StandardError
    nil
  end
  lines << 'POLICY (R5 tabular Q / REINFORCE — advisory only, does not replace planning)'
  lines << "  episodes=#{s[:n_episodes]} states=#{s[:n_states]} pairs=#{s[:n_pairs]} updates=#{s[:n_updates]}"
  lines << "  mean_return=#{s[:mean_return] || '-'} mean|TD|=#{s[:mean_abs_td] || '-'} greedy_match=#{ev[:greedy_match] || '-'}"
  lines << "  current_state=#{current_state || '(none)'} suggest=#{rec || '-'} tool_preference=#{actions.join(',')}"
  "#{lines.join("\n")}\n"
rescue StandardError
  ''
end

.trajectories(opts = {}) ⇒ Object 



589
590
591
592
593
594
595
596
597
598
599
600
 
# File 'lib/pwn/ai/agent/policy.rb', line 589

public_class_method def self.trajectories(opts = {})
  limit = (opts[:limit] || 50).to_i
  return [] unless File.exist?(TRAJECTORY_FILE)

  File.readlines(TRAJECTORY_FILE).last(limit).filter_map do |line|
    JSON.parse(line, symbolize_names: true)
  rescue StandardError
    nil
  end.reverse
rescue StandardError
  []
end

.update_pg!(opts = {}) ⇒ Object

Supported Method Parameters

h = PWN::AI::Agent::Policy.update_pg!( state: 'required', action: 'required', advantage: 'required - scalar G_t − V(s)' )



425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
 
# File 'lib/pwn/ai/agent/policy.rb', line 425

public_class_method def self.update_pg!(opts = {})
  return nil unless enabled?

  s = opts[:state].to_s
  a = opts[:action].to_s
  adv = opts[:advantage].to_f
  return nil if s.empty? || a.empty?
  return 0.0 if adv.abs < 1e-9

  tab = load
  logits = (tab[:h][s.to_sym] || {}).dup
  logits[a.to_sym] = logits[a.to_sym].to_f
  pi = softmax(logits: logits)
  logits.each_key do |act|
    grad = act.to_s == a ? (1.0 - pi[act].to_f) : -pi[act].to_f
    logits[act] = logits[act].to_f + (ALPHA_PG * adv * grad)
  end
  tab[:h][s.to_sym] = logits
  save(table: tab)
  logits[a.to_sym].to_f.round(5)
rescue StandardError => e
  warn "[pwn-ai/policy] update_pg! swallowed: #{e.class}: #{e.message}"
  nil
end

.update_q!(opts = {}) ⇒ Object

Supported Method Parameters

q = PWN::AI::Agent::Policy.update_q!( transition: 'required - Hash with :state :action :reward :next_state :terminal' )



390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
 
# File 'lib/pwn/ai/agent/policy.rb', line 390

public_class_method def self.update_q!(opts = {})
  return nil unless enabled?

  tr = opts[:transition] || opts
  s  = tr[:state].to_s
  a  = tr[:action].to_s
  return nil if s.empty? || a.empty?

  r  = tr[:reward].to_f
  s2 = tr[:next_state].to_s
  term = tr[:terminal] ? true : false
  tab = load
  qsa = read_q(table: tab, state: s, action: a)
  max_n = term ? 0.0 : max_q(table: tab, state: s2)
  target = r + (GAMMA * max_n)
  td = target - qsa
  new_q = qsa + (ALPHA * td)
  write_q!(table: tab, state: s, action: a, value: new_q)
  bump_visit!(table: tab, state: s, action: a)
  tab[:n_updates] = tab[:n_updates].to_i + 1
  tab[:td_abs_sum] = tab[:td_abs_sum].to_f + td.abs
  save(table: tab)
  new_q.round(5)
rescue StandardError => e
  warn "[pwn-ai/policy] update_q! swallowed: #{e.class}: #{e.message}"
  nil
end

.value(opts = {}) ⇒ Object 



460
461
462
463
464
 
# File 'lib/pwn/ai/agent/policy.rb', line 460

public_class_method def self.value(opts = {})
  max_q(table: load, state: opts[:state])
rescue StandardError
  0.0
end

.warm? ⇒ Boolean 



141
142
143
144
145
 
# File 'lib/pwn/ai/agent/policy.rb', line 141

public_class_method def self.warm?
  stats[:n_episodes].to_i >= WARM_EPISODES
rescue StandardError
  false
end

.warmup!(opts = {}) ⇒ Object

Replay stored trajectories into Q so a cold table is not empty advice.



1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
 
# File 'lib/pwn/ai/agent/policy.rb', line 1313

public_class_method def self.warmup!(opts = {})
  return { skipped: :disabled } unless enabled?
  return { skipped: :no_traj } unless File.exist?(TRAJECTORY_FILE)

  tab = load
  rows = trajectories(limit: opts[:limit] || 400).reject { |ep| ep[:score].nil? }
  context_counts = Hash.new(0)
  rows.each do |ep|
    Array(ep[:steps]).each do |tr|
      next if tr[:context_state].to_s.empty? || tr[:action].to_s.empty?
      next if transition_variants(transition: tr).empty?

      context_counts[[tr[:context_state].to_s.to_sym, tr[:action].to_s.to_sym]] += 1
    end
  end
  n = 0
  2.times do
    rows.reverse_each do |ep|
      Array(ep[:steps]).flat_map { |tr| transition_variants(transition: tr) }.each do |tr|
        s = tr[:state].to_s
        a = tr[:action].to_s
        next if s.empty? || a.empty?

        r = tr[:reward].to_f
        s2 = tr[:next_state].to_s
        term = tr[:terminal] ? true : false
        qsa = read_q(table: tab, state: s, action: a)
        max_n = term ? 0.0 : max_q(table: tab, state: s2)
        target = r + (GAMMA * max_n)
        td = target - qsa
        write_q!(table: tab, state: s, action: a, value: qsa + (ALPHA * td))
        bump_visit!(table: tab, state: s, action: a) unless s.include?('~ctx:')
        tab[:n_updates] = tab[:n_updates].to_i + 1
        tab[:td_abs_sum] = tab[:td_abs_sum].to_f + td.abs
        n += 1
      end
    end
  end
  # Replay is not new evidence: restore observed counts, never count
  # the two passes (or repeated warmups) as independent samples.
  context_counts.each do |(s, a), count|
    tab[:visits][s] ||= {}
    tab[:visits][s][a] = [tab[:visits][s][a].to_i, count].max
  end
  # Credit stored returns toward the episode budget so greedy
  # suggestions are not omitted after a successful replay of a
  # table that never finished COLD_EPISODES live turns.
  rets = Array(tab[:returns])
  need = COLD_EPISODES - rets.length
  if need.positive?
    extras = rows.filter_map { |ep| ep[:return] unless ep[:return].nil? }.first(need)
    tab[:returns] = (rets + extras).last(200)
  end
  tab[:warmed_at] = Time.now.utc.iso8601
  save(table: tab)
  { replayed: rows.length, td_updates: n, n_episodes: Array(load[:returns]).length }
rescue StandardError => e
  { error: "#{e.class}: #{e.message}" }
end