|
|
| """
|
| Download, transform LMSYS-Chat-1M into plain text for LLM completion models
|
| in the format:
|
| <|im_start|>role
|
| message<|endoftext|>
|
| <|im_stop|>
|
|
|
| with 6 newlines between conversations.
|
| """
|
|
|
| from datasets import load_dataset
|
| import sys
|
|
|
| def main(output_path="lmsys_chat_1m.txt", split="train"):
|
| ds = load_dataset("lmsys/lmsys-chat-1m", split=split)
|
|
|
| with open(output_path, "w", encoding="utf-8") as out:
|
| for i, sample in enumerate(ds):
|
| conv = sample["conversation"]
|
|
|
| for msg in conv:
|
| role = msg["role"]
|
| content = msg["content"].strip()
|
| out.write(f"<|im_start|>{role}\n{content}<|endoftext|>\n<|im_stop|>\n")
|
|
|
| out.write("\n" * 6)
|
|
|
| if (i + 1) % 10000 == 0:
|
| print(f"Processed {i + 1} conversations", file=sys.stderr)
|
|
|
| print(f"✔ Saved plain-text to: {output_path}")
|
|
|
| if __name__ == "__main__":
|
| import argparse
|
| p = argparse.ArgumentParser(description="Convert LMSYS-Chat-1M to LLM-friendly text format")
|
| p.add_argument("--output", "-o", default="lmsys_chat_1m.txt", help="Output file path")
|
| p.add_argument("--split", "-s", default="train", help="Dataset split (e.g. 'train')")
|
| args = p.parse_args()
|
| main(output_path=args.output, split=args.split)
|
|
|
