Repository navigation
Expand file tree
/
Copy pathgenerate_utils.py
More file actions
2617 lines (2226 loc) · 99.1 KB
/
Copy pathgenerate_utils.py
File metadata and controls
2617 lines (2226 loc) · 99.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
"""Utility functions for generating text with various models and strategies."""
import json
import os
import random
from concurrent.futures import ThreadPoolExecutor, as_completed
from copy import deepcopy
from dataclasses import dataclass
import re
from typing import Dict, List, Optional, Union
import requests
import torch
from dotenv import load_dotenv
from openai import OpenAI
from rich import box
from rich.console import Console
from rich.panel import Panel
from rich.table import Table
from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential, wait_random_exponential
from tqdm import tqdm
from cp_eval_utils import (
anonymize_reasonings_parallel,
calculate_openrouter_cost,
swap_reasonings_parallel,
)
@dataclass
class OutputObj:
"""Simple dataclass to mimic VLLM's output structure.
Attributes
----------
text : str
The generated text output.
"""
text: str
@dataclass
class RequestOutputObj:
"""Dataclass to mimic VLLM's RequestOutput structure.
Attributes
----------
outputs : List[OutputObj]
A list of output objects, each containing generated text.
prompt : Union[str, List[Dict]]
The prompt used to generate the output.
"""
outputs: List[OutputObj]
prompt: Union[str, List[Dict]]
class UserDataLogitsProcessor:
"""A logits processor that blocks generation of user data tokens.
This processor is used during the model's "thinking" phase to prevent it
from leaking personally identifiable information (PII) or other sensitive
user data that was part of the input prompt. It works by assigning a
log-probability of -inf to token IDs corresponding to the user's data,
effectively blocking them from being generated. The blocking is deactivated
once an `end_think_token` is generated.
Attributes
----------
tokenizer : PreTrainedTokenizer
The tokenizer used to encode text into token IDs.
user_data : dict or list
A nested structure containing user data to be blocked.
end_think_token : str, optional
The token that signals the end of the thinking phase. If None, blocking
is always active.
end_think_token_ids : list of int, optional
The token IDs for the `end_think_token`.
is_thinking_phase : bool
A flag indicating whether the model is currently in the thinking phase.
blocked_token_ids : set of int
A set of token IDs that are blocked from being generated.
"""
def __init__(self, tokenizer, user_data, end_think_token=None):
"""Initialize the UserDataLogitsProcessor.
Parameters
----------
tokenizer : PreTrainedTokenizer
The tokenizer for encoding user data.
user_data : dict or list
The user data to block during generation.
end_think_token : str, optional
The string marking the end of the thinking phase. Default is None.
"""
self.tokenizer = tokenizer
self.user_data = user_data
self.end_think_token = end_think_token
self.end_think_token_ids = (
None
if end_think_token is None
else tokenizer.encode(end_think_token, add_special_tokens=False)
)
self.is_thinking_phase = True
# Pre-compute token IDs for all user data values
self.blocked_token_ids = set()
self.parsed_user_data = self._extract_values(user_data)
# Get all values from the profile
values = [
str(v)
for v in self.parsed_user_data
if isinstance(v, (str, int, float, bool))
]
values = [
[v, " " + v, v.lower(), " " + v.lower(), v.upper(), " " + v.upper()]
for v in values
]
values = list(set([item for sublist in values for item in sublist]))
token_ids = [self.tokenizer.encode(v, add_special_tokens=False) for v in values]
token_ids = list(set([item for sublist in token_ids for item in sublist]))
self.blocked_token_ids.update(token_ids)
def _extract_values(self, data):
"""Recursively extract all values from nested dictionaries and lists.
Parameters
----------
data : dict or list
The data structure to extract values from.
Returns
-------
list
A flat list of all values found in the data structure.
"""
values = []
if isinstance(data, dict):
for value in data.values():
values.extend(self._extract_values(value))
elif isinstance(data, list):
for item in data:
values.extend(self._extract_values(item))
else:
values.append(data)
return values
def __call__(self, input_ids, logits):
"""Process logits to block user data tokens.
This method is called at each generation step. It modifies the logits
to prevent the generation of blocked tokens during the thinking phase.
Parameters
----------
input_ids : torch.Tensor
The sequence of input IDs generated so far.
logits : torch.Tensor
The logits for the next token.
Returns
-------
torch.Tensor
The modified logits.
"""
if (
self.end_think_token_ids is not None
and self.is_thinking_phase
and len(input_ids) > 1
):
last_tokens = input_ids[-len(self.end_think_token_ids) :]
think_token_match = torch.equal(
torch.tensor(last_tokens, device=logits.device),
torch.tensor(self.end_think_token_ids, device=logits.device),
)
if think_token_match:
self.is_thinking_phase = False
return logits
# Only block tokens during thinking phase
if self.is_thinking_phase:
for token_id in self.blocked_token_ids:
logits[token_id] = float("-inf")
return logits
def generate_with_openrouter(
prompts, model_name, sampling_params, args, end_think_token=None, is_cot=False
):
"""Generate text using the OpenRouter API.
This function sends prompts to the OpenRouter API for text generation,
handling parallel requests, retries, and cost calculation. It's designed
to work with models available through OpenRouter, such as DeepSeek-R1.
Parameters
----------
prompts : list of list of dict
A list of prompts, where each prompt is a list of messages in chat format.
model_name : str
The name of the model to use on OpenRouter (e.g., 'deepseek/deepseek-chat').
sampling_params : object
An object containing sampling parameters like temperature, top_p, max_tokens.
args : argparse.Namespace
Command-line arguments, expected to contain `openrouter_settings`.
end_think_token : str, optional
The token that separates reasoning from the final answer. If provided,
the two parts are concatenated. Default is None.
is_cot : bool, optional
Flag indicating if it is a Chain-of-Thought prompt. Default is False.
Returns
-------
list of RequestOutputObj
A list of output objects, each containing the generated text and original prompt.
"""
print("Starting generation with OpenRouter API, generate_with_openrouter")
# Load API key from .env file
load_dotenv()
api_key = os.getenv("OPENROUTER_API_KEY")
if not api_key:
raise ValueError("OPENROUTER_API_KEY not found in .env file")
# Load OpenRouter settings
try:
with open(args.openrouter_settings, "r") as f:
openrouter_settings = json.load(f)
except FileNotFoundError:
print(
f"Warning: OpenRouter settings file {args.openrouter_settings} not found. Using default settings."
)
openrouter_settings = {
"provider": {
"order": ["DeepInfra"],
"allow_fallbacks": False,
"require_parameters": True,
"data_collection": "deny",
}
}
if (
model_name == "deepseek/deepseek-chat"
): # for some reason DeepInfra does not take tool outputs
openrouter_settings["provider"].pop("order")
openrouter_settings["provider"]["allow_fallbacks"] = True
all_outputs = [None] * len(prompts) # Initialize with correct size
num_workers = min(50, len(prompts)) # Number of parallel workers
generation_ids = [] # Store all generation IDs
generation_id_to_prompt_idx = {} # Map generation IDs to prompt indices
print(
f"Generating responses with OpenRouter API for {len(prompts)} prompts using {num_workers} workers..."
)
@retry(
stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4, max=10)
)
def make_api_request(params, prompt):
"""Make a single API request to OpenRouter with retries."""
headers = {
"Authorization": f"Bearer {api_key}",
"HTTP-Referer": "https://github.com/leaking_thoughts",
"X-Title": "Leaking Thoughts",
}
response = requests.post(
url="https://openrouter.ai/api/v1/chat/completions",
headers=headers,
json={**params, "messages": prompt},
)
response.raise_for_status()
response_data = response.json()
return response_data["choices"][0]["message"], response_data["id"]
def process_single_prompt(prompt, prompt_idx, end_think_token=None, is_cot=False):
"""Process a single prompt to generate `n` samples."""
batch_outputs = []
for _ in range(sampling_params.n):
# Set up generation parameters
params = {
"model": model_name,
"max_tokens": sampling_params.max_tokens,
"temperature": sampling_params.temperature,
}
if sampling_params.top_p is not None:
params["top_p"] = sampling_params.top_p
if hasattr(sampling_params, "stop") and sampling_params.stop:
params["stop"] = sampling_params.stop
# Add OpenRouter settings
params.update(openrouter_settings)
# Make API request with retry mechanism
response_output, gen_id = make_api_request(params, prompt)
if (
end_think_token is not None
and not is_cot
and "reasoning" in response_output
and "content" in response_output
and response_output["reasoning"] is not None
and response_output["content"] is not None
):
output_text = (
response_output["reasoning"]
+ end_think_token
+ response_output["content"]
)
else:
output_text = response_output["content"]
generation_ids.append(gen_id)
generation_id_to_prompt_idx[gen_id] = prompt_idx
# Create object that mimics VLLM's output structure
batch_outputs.append(OutputObj(output_text))
return prompt_idx, batch_outputs
# Process prompts in parallel
with ThreadPoolExecutor(max_workers=num_workers) as executor:
# Submit all tasks
future_to_idx = {
executor.submit(
process_single_prompt, prompt, i, end_think_token, is_cot
): i
for i, prompt in enumerate(prompts)
}
# Create progress bar
progress_bar = tqdm(total=len(prompts), desc="OpenRouter API calls")
# Process results as they complete
for future in as_completed(future_to_idx):
idx, batch_outputs = future.result()
# Create an object that mimics VLLM's RequestOutput structure and place it at the correct index
all_outputs[idx] = RequestOutputObj(batch_outputs, prompts[idx])
# Update progress bar
progress_bar.update(1)
print(f"Completed {len(all_outputs)} OpenRouter API calls")
# Calculate and display total cost
total_cost, provider_info = calculate_openrouter_cost(generation_ids, api_key)
cost_console = Console()
cost_panel = Panel(
f"[bold white]Total OpenRouter API Cost:[/] [bold green]${total_cost:.2f}[/]",
title="💰 Cost Summary",
border_style="green",
)
cost_console.print()
cost_console.print(cost_panel)
cost_console.print()
# Add provider info to outputs
for gen_id, prompt_idx in generation_id_to_prompt_idx.items():
if not hasattr(all_outputs[prompt_idx], "provider_info"):
all_outputs[prompt_idx].provider_info = []
all_outputs[prompt_idx].provider_info.append(provider_info[gen_id])
return all_outputs
def get_provider_model_name(model_name, provider):
"""Get the correct model name format for the specified provider.
Different providers (OpenRouter, DeepSeek API, local vLLM) may use
different identifiers for the same model. This function canonicalizes
the model name based on the specified provider.
Parameters
----------
model_name : str
The generic model name (e.g., 'deepseek-ai/deepseek-r1').
provider : str
The provider name ('openrouter', or 'vllm').
Returns
-------
str
The provider-specific model name.
Raises
------
ValueError
If a model is not supported by the specified provider.
"""
# Handle DeepSeek model naming conventions per provider
if model_name.lower() in [
"deepseek-ai/deepseek-r1",
"deepseek/deepseek-r1",
]:
if provider == "openrouter":
return "deepseek/deepseek-r1"
elif provider == "vllm":
raise ValueError(
"Cannot use vLLM as provider, as models cannot be run locally. Please use 'openrouter' or 'deepseek' as provider."
)
elif model_name.lower() == "deepseek-ai/deepseek-v3":
if provider == "openrouter":
return "deepseek/deepseek-chat"
elif provider == "vllm":
raise ValueError(
"Cannot use vLLM as provider, as models cannot be run locally. Please use 'openrouter' as provider."
)
elif model_name.lower() == "deepseek-ai/deepseek-v3-0324":
if provider == "openrouter":
return "deepseek/deepseek-chat-v3-0324"
elif provider == "vllm":
raise ValueError(
"Cannot use vLLM as provider, as models cannot be run locally. Please use 'openrouter' or 'deepseek' as provider."
)
return model_name
def display_generation_config(console, sampling_params):
"""Display the generation configuration in a pretty table.
Parameters
----------
console : rich.console.Console
The rich console object for printing.
sampling_params : object
An object containing the sampling parameters for generation.
Returns
-------
dict
A dictionary containing the generation configuration parameters.
"""
# Save sampling parameters in a gen_conf dictionary
gen_conf = {
"temperature": sampling_params.temperature
if hasattr(sampling_params, "temperature")
else None,
"top_p": sampling_params.top_p if hasattr(sampling_params, "top_p") else None,
"top_k": sampling_params.top_k if hasattr(sampling_params, "top_k") else None,
"repetition_penalty": sampling_params.repetition_penalty
if hasattr(sampling_params, "repetition_penalty")
else None,
"max_tokens": sampling_params.max_tokens,
"n": sampling_params.n,
"seed": sampling_params.seed,
"stop": sampling_params.stop if hasattr(sampling_params, "stop") else None,
"skip_special_tokens": sampling_params.skip_special_tokens
if hasattr(sampling_params, "skip_special_tokens")
else None,
}
# Pretty print the generation configuration using rich
gen_conf_table = Table(title="Generation Configuration", box=box.ROUNDED)
gen_conf_table.add_column("Parameter", style="cyan")
gen_conf_table.add_column("Value", style="green")
for param, value in gen_conf.items():
gen_conf_table.add_row(param, str(value))
console.print()
console.print(Panel(gen_conf_table, expand=False))
console.print()
return gen_conf
_SUMMARIZE_DEBIAS_CODE_SYSTEM = """
##Definition:
"Social Bias of LLMs in Code Generation" refers to the phenomenon where large language models (LLMs), when generating code, intentionally or unintentionally exhibit unfair or discriminatory tendencies in contexts involving social attributes such as gender, age, race, occupation, or social roles.
##Your goal:
You analyze text about code or programming.
Delete any sentence or phrase that exhibits unfair or discriminatory tendencies toward social attributes from the given text, regarding given the task.
Keep all other technical, neutral, or factual content unchanged.
## Instrcutions:
Perform two steps on the INPUT TEXT:
Step 1 (Summarize): Summarize the given chain-of-thought into the concise, information-dense reasoning
(purpose, behavior, inputs/outputs, APIs, constraints) descriptions. Keep the key information as much as possible. **DON NOT speculation** and *DON NOT add your own thoughts*.
Step 2 (Debias): For the summarized reasoning descriptions, Remove potentially biased/subjective/opinionated language objective and verifiable statements about the code.
## Important: Output ONLY the cleaned reasoning text.
"""
def _build_openai_client():
base_url = os.getenv("OPENAI_BASE_URL", None)
from openai import OpenAI
if base_url:
return OpenAI(api_key=os.getenv("OPENAI_API_KEY"), base_url=base_url)
return OpenAI(api_key=os.getenv("OPENAI_API_KEY"))
def _build_code_user_prompt(thinking_text: str) -> str:
return (
f"Original chain-of-thought (between special tags):\n{thinking_text}"
)
@retry(
reraise=True,
stop=stop_after_attempt(3),
wait=wait_random_exponential(multiplier=1, max=10),
retry=retry_if_exception_type(Exception),
)
def _summarize_and_debias_code_once(
client,
model: str,
text: str,
max_tokens: int = 10000
) -> str:
resp = client.chat.completions.create(
model=model,
temperature=0,
max_tokens=max_tokens,
messages=[
{"role": "system", "content": _SUMMARIZE_DEBIAS_CODE_SYSTEM},
{"role": "user", "content": _build_code_user_prompt(text)},
],
)
text = (resp.choices[0].message.content or "").strip()
# 简单清理 code fence(以防万一)
if text.startswith("```"):
text = re.sub(r"^```[a-zA-Z0-9_+-]*\n?", "", text)
text = re.sub(r"\n?```$", "", text).strip()
return text
def summarize_and_sanitize_reasonings(
think_texts: List[str],
openai_model: str = "gpt-4o-mini",
max_tokens: int = 512,
parallel: bool = True,
) -> List[str]:
"""
用 ChatGPT 对思考文本做摘要 + 去bias。默认串行,想并行可把 parallel=True 并用线程池。
"""
client = _build_openai_client()
if not os.getenv("OPENAI_API_KEY"):
# 没有 key 就“无操作”返回,以免流程中断
raise ValueError("OPENAI_API_KEY not found in .env file")
results = []
if parallel:
import concurrent.futures as cf
with cf.ThreadPoolExecutor(max_workers=min(8, len(think_texts))) as ex:
futs = [
ex.submit(_summarize_and_debias_code_once, client, openai_model, t, max_tokens)
for t in think_texts
]
for fut in futs:
try:
results.append(fut.result() or "")
except Exception:
results.append("")
else:
for t in think_texts:
try:
results.append(_summarize_and_debias_code_once(client, openai_model, t, max_tokens))
except Exception:
results.append("")
# 兜底:若某些为空,则回退原文
cleaned = [r if r.strip() else t for r, t in zip(results, think_texts)]
return cleaned
# -----------------------------------------------------------
# 自由生成思考(到 </think> 截止)——无 budget 循环
# -----------------------------------------------------------
def _gen_reasoning_free(
llm,
prompts,
sampling_params,
args,
start_think_token: str,
end_think_token: str,
is_qwen3_or_r1: bool,
tokenizer,
custom_template: Optional[str],
) -> List[str]:
reason_params = sampling_params.clone()
# 建议:给思考阶段一个单独上限(不走 budget,但要止损)
reason_params.max_tokens = 2048
reason_params.stop = [end_think_token, " " + end_think_token]
reason_params.skip_special_tokens = False
reason_params.include_stop_str_in_output = True
if is_qwen3_or_r1:
# render 到 <think> 起点,再用 llm.generate
rendered = []
for msgs in prompts:
s = _apply_template_to_str(tokenizer, msgs, enable_thinking=True, add_generation_prompt=True)
if s.rstrip().endswith(start_think_token):
s += "\n"
rendered.append(s)
outs = llm.generate(rendered, sampling_params=reason_params)
texts = [o.outputs[0].text for o in outs]
else:
outs = llm.chat(
prompts,
sampling_params=reason_params,
chat_template=custom_template if custom_template is not None else tokenizer.chat_template,
add_generation_prompt=False, # reasoning 阶段通常不需要再添加 <assistant> 引导
continue_final_message=True, # 让模型继续在最后一条消息上写
)
texts = [o.outputs[0].text for o in outs]
# 强制闭合,保证下游流程不崩
cleaned = []
for t in texts:
cleaned.append(_ensure_close_with_phrase(t, end_think_token))
return cleaned
# -----------------------------------------------------------
# 主流程:RAnA(自由生成 → 摘要去敏 → 拼回作答)
# -----------------------------------------------------------
def generate_with_rana(
llm,
prompts,
args,
model_name,
start_think_token,
end_think_token,
sampling_params=None,
):
"""
自由生成思考(到 </think> 截止),把思考交给 ChatGPT 做“摘要+去敏”,
再把处理后的 think 拼回本地模型继续生成答案。
"""
print("Starting RAnA generation process (free reasoning)")
tokenizer = llm.get_tokenizer()
is_qwen3 = _is_qwen3_thinker(args)
is_deep_r1 = _is_deepseek_r1_thinker(args)
# 自定义模板(例如 deepseek 等需要保留思考内容的模板)
custom_template_path = f"chat_templates/rana/{model_name.replace('/', '_')}.jinja"
try:
with open(custom_template_path, "r", encoding="utf-8") as f:
custom_template = f.read()
print(f"Using custom chat template from {custom_template_path}")
except FileNotFoundError:
custom_template = None
print(f"No custom template for {model_name}. Using default tokenizer.chat_template")
# ---- Step 1: 自由生成“思考”到 </think> ----
final_reasonings = _gen_reasoning_free(
llm=llm,
prompts=prompts,
sampling_params=sampling_params,
args=args,
start_think_token=start_think_token,
end_think_token=end_think_token,
is_qwen3_or_r1=is_qwen3 or is_deep_r1,
tokenizer=tokenizer,
custom_template=custom_template,
)
# (可选)把 </think> 去掉再送给 OpenAI(一般去掉更干净)
stripped_reasonings = [r.replace(end_think_token, "").strip() for r in final_reasonings]
# -------------------------------------------------------
# Step B: 用 ChatGPT 做“摘要 + 去敏/去偏”
# - 如未设置 OPENAI_API_KEY,则回退原文
# - 这里无 profile 时传 None
# -------------------------------------------------------
openai_model = getattr(args, "gpt_eval_model", "")
openai_max_tokens = getattr(args, "openai_max_tokens", 10000)
cleaned_reasonings = summarize_and_sanitize_reasonings(
think_texts=stripped_reasonings,
openai_model=openai_model,
max_tokens=openai_max_tokens,
parallel=True,
)
# 确保闭合:给清洗后的文本补上 </think>(规范闭合,避免模板截断)
closed_reasonings = [_ensure_close_with_phrase(cr, end_think_token) for cr in cleaned_reasonings]
# -------------------------------------------------------
# Step C: 把清洗后的思考拼回 prompt,让本地模型生成“最终答案”
# - Qwen3:渲染到 <think> 起点 → 拼接 cleaned + </think> → llm.generate
# - 非 Qwen:直接在 messages 里 append assistant 一条包含 cleaned + </think> → llm.chat
# -------------------------------------------------------
answer_params = sampling_params.clone() if sampling_params is not None else None
if answer_params is None:
from vllm import SamplingParams
answer_params = SamplingParams()
answer_params.max_tokens = args.max_tokens - 2048
# 答案阶段通常不要再 stop 掐 </think>;只在 reasoning 阶段使用 stop
answer_params.stop = None
answer_params.include_stop_str_in_output = False
answer_params.skip_special_tokens = True
outputs = []
if is_qwen3 or is_deep_r1:
# ---- Qwen3:render 到 <think> 起点,再拼 cleaned + </think> ----
rendered_with_reason = []
for i, msgs in enumerate(prompts):
s = _apply_template_to_str(tokenizer, msgs, enable_thinking=True, add_generation_prompt=True)
# 如果刚好以 <think> 结尾,补个换行更自然
if s.rstrip().endswith(start_think_token):
s += "\n"
# 拼回清洗后的 reasoning,并强制闭合
s += closed_reasonings[i]
rendered_with_reason.append(s)
ans_outs = llm.generate(rendered_with_reason, sampling_params=answer_params)
for i, ans in enumerate(ans_outs):
answer_text = ans.outputs[0].text
# 防止答案里多余地再复述 </think>
if answer_text.startswith(end_think_token):
answer_text = answer_text[len(end_think_token):]
if answer_text.endswith(end_think_token):
answer_text = answer_text[: -len(end_think_token)]
ans.outputs[0].text = closed_reasonings[i] + answer_text
ans.prompt = final_reasonings[i]
outputs.append(ans)
else:
# ---- 非 Qwen:直接在对话最后追加“assistant: <think>... </think>” ----
answer_prompts = []
for i, msgs in enumerate(prompts):
new_msgs = deepcopy(msgs)
new_msgs[-1]["content"] += closed_reasonings[i]
answer_prompts.append(new_msgs)
ans_outs = llm.chat(
answer_prompts,
sampling_params=answer_params,
chat_template=custom_template if custom_template is not None else tokenizer.chat_template,
add_generation_prompt=False,
continue_final_message=True, # 继续在 assistant 的最后一条消息后写“答案”
)
for i, ans in enumerate(ans_outs):
answer_text = ans.outputs[0].text
if answer_text.startswith(end_think_token):
answer_text = answer_text[len(end_think_token):]
if answer_text.endswith(end_think_token):
answer_text = answer_text[: -len(end_think_token)]
ans.outputs[0].text = closed_reasonings[i] + answer_text
ans.prompt = final_reasonings[i]
outputs.append(ans)
return outputs
def generate_with_openrouter_rana(
prompts,
data,
valid_indices,
model_name,
sampling_params,
args,
start_think_token,
end_think_token,
):
"""Implement the Reason-Anonymize-Answer (RAnA) approach using the OpenRouter API.
This function orchestrates the RAnA pipeline with OpenRouter as the backend:
1. Generate reasoning in parallel for each prompt, stopping at `end_think_token`.
2. Anonymize the generated reasoning traces to remove PII.
3. Feed the anonymized reasoning back to the OpenRouter API to generate final answers.
Parameters
----------
prompts : list
A list of prompts for the model.
data : list of dict
The dataset, containing user profile data for each prompt.
valid_indices : list of int
The indices of the prompts/data to be processed.
model_name : str
The name of the model to use on OpenRouter.
sampling_params : object
An object with sampling parameters (temperature, top_p, etc.).
args : argparse.Namespace
Command-line arguments, containing model path and prompt type.
start_think_token : str
The token to prepend to the reasoning.
end_think_token : str
The token to signal the end of the reasoning phase.
Returns
-------
tuple
- list of RequestOutputObj: The final generated outputs.
- list of str: The generation IDs from OpenRouter.
- dict: A mapping from generation IDs to prompt indices.
"""
import time
from copy import deepcopy
from transformers import AutoTokenizer
# Load API key from .env file
load_dotenv()
api_key = os.getenv("OPENROUTER_API_KEY")
if not api_key:
raise ValueError("OPENROUTER_API_KEY not found in .env file")
# Load OpenRouter settings
try:
with open(args.openrouter_settings, "r") as f:
openrouter_settings = json.load(f)
except FileNotFoundError:
print(
f"Warning: OpenRouter settings file {args.openrouter_settings} not found. Using default settings."
)
openrouter_settings = {
"provider": {
"order": ["DeepInfra"],
"allow_fallbacks": False,
"require_parameters": True,
"data_collection": "deny",
}
}
if model_name == "deepseek/deepseek-chat":
openrouter_settings["provider"].pop("order")
openrouter_settings["provider"]["allow_fallbacks"] = True
# Initialize variables to store generation results
reasoning_texts = [None] * len(valid_indices) # Initialize with correct size
num_workers = min(50, len(valid_indices)) # Number of parallel workers
generation_ids = [] # Store all generation IDs
generation_id_to_prompt_idx = {} # Map generation IDs to prompt indices
print(
f"Generating responses with OpenRouter API for {len(valid_indices)} prompts using {num_workers} workers in RAnA mode..."
)
# Load tokenizer for applying chat templates
tokenizer = AutoTokenizer.from_pretrained(args.model)
# Path to custom chat template
custom_template_path = f"chat_templates/rana/{args.model.replace('/', '_')}.jinja"
# Load custom chat template
try:
with open(custom_template_path, "r") as f:
custom_template = f.read()
tokenizer.chat_template = custom_template
print(f"Using custom chat template from {custom_template_path}")
except FileNotFoundError:
print(f"Custom template not found for {args.model} at {custom_template_path}")
print("Using default chat template")
@retry(
stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4, max=10)
)
def make_api_request(params, prompt_text):
headers = {
"Authorization": f"Bearer {api_key}",
"HTTP-Referer": "https://github.com/leaking_thoughts",
"X-Title": "Leaking Thoughts",
}
# Always use completions endpoint
response = requests.post(
url="https://openrouter.ai/api/v1/completions",
headers=headers,
json={**params, "prompt": prompt_text},
)
response.raise_for_status()
gen_id = response.json()["id"]
output = response.json()
return output, gen_id
# Step 1: Generate reasoning for each prompt in parallel
# Max tokens for reasoning is max_tokens - 500
reasoning_max_tokens = max(sampling_params.max_tokens - 500, 1000)
print(
f"Step 1: Generating reasoning in parallel (max tokens: {reasoning_max_tokens})..."
)
# Function to process a single reasoning prompt
def process_reasoning_prompt(prompt_idx):
idx = valid_indices[prompt_idx]
prompt = prompts[idx]
# Format the prompt using the chat template if it's a list (chat format)
if isinstance(prompt, list):
formatted_prompt = tokenizer.apply_chat_template(
prompt,
tokenize=False,
add_generation_prompt=False if "cot" in args.prompt_type else True,
continue_final_message=True if "cot" in args.prompt_type else False,
)
else:
# For non-chat prompts, use as-is
formatted_prompt = prompt
# Set up generation parameters for reasoning
reasoning_params = {
"model": model_name,
"max_tokens": reasoning_max_tokens,
"temperature": sampling_params.temperature,
}
if sampling_params.top_p is not None:
reasoning_params["top_p"] = sampling_params.top_p
# Add stop tokens to end at reasoning phase
if end_think_token is not None:
reasoning_params["stop"] = [end_think_token, " " + end_think_token]
# Add OpenRouter settings
reasoning_params.update(openrouter_settings)
# Make API request for reasoning
response_output, gen_id = make_api_request(reasoning_params, formatted_prompt)
reasoning_key = "reasoning" if "reasoning" in args.prompt_type else "text"
reasoning_text = response_output["choices"][0][reasoning_key]
# Add end_think_token if needed
if (
end_think_token is not None
and reasoning_text is not None
and not reasoning_text.endswith(end_think_token)
):
reasoning_text += end_think_token
# Store generation ID mapping
return prompt_idx, idx, reasoning_text, gen_id, formatted_prompt
# Process reasoning prompts in parallel
with ThreadPoolExecutor(max_workers=num_workers) as executor:
# Submit all reasoning tasks
future_to_idx = {
executor.submit(process_reasoning_prompt, i): i
for i in range(len(valid_indices))
}
# Create progress bar
progress_bar = tqdm(
total=len(valid_indices), desc="Step 1: Reasoning generation"
)
# Process results as they complete
for future in as_completed(future_to_idx):
prompt_idx, data_idx, reasoning_text, gen_id, formatted_prompt = (
future.result()
)
reasoning_texts[prompt_idx] = reasoning_text
# Store generation ID information
generation_ids.append(gen_id)
generation_id_to_prompt_idx[gen_id] = data_idx
# Update progress bar
progress_bar.update(1)
# Close progress bar
progress_bar.close()
# Step 2: Anonymize all reasoning texts in parallel
print("Step 2: Anonymizing reasoning in parallel...")
# Get a representative profile for anonymization
# Using the first valid index's profile as a representative
sample_profile = data[valid_indices[0]].get("profile", {})
# Anonymize all reasoning texts in parallel
anonymized_results = anonymize_reasonings_parallel(reasoning_texts, sample_profile)
# Store anonymized reasoning and extracted PII in data
anonymized_reasoning_list = []
for i, idx in enumerate(valid_indices):
anonymized_text, extracted_pii = anonymized_results[i]
# Store original reasoning in data
data[idx]["original_reasoning"] = reasoning_texts[i]
# Store extracted PII data
if "gpt_extractions" not in data[idx]:
data[idx]["gpt_extractions"] = {}
data[idx]["gpt_extractions"]["reasoning"] = extracted_pii
# Add to anonymized list for next step
anonymized_reasoning_list.append(anonymized_text)
# Step 3: Generate answers based on anonymized reasoning in parallel
final_outputs = [None] * len(valid_indices) # Initialize with correct size
print("Step 3: Generating answers in parallel (max tokens: 500)...")
# Max tokens for answer generation is fixed at 500
answer_max_tokens = 500
# Function to process a single answer generation
def process_answer_prompt(prompt_idx):
idx = valid_indices[prompt_idx]
orig_prompt = prompts[idx]