tokenizer.py

from transformers import GPT2Tokenizer

tokenizer = GPT2Tokenizer.from_pretrained("gpt2")

file_path = "/Users/minapetrovic/Music/thesis/256_medicine/train_256_medicine_balanced.txt"

samples = []
current_sample = ""

with open(file_path, "r", encoding="utf-8") as f:
    for line in f:
        line = line.rstrip()

        if "<|startoftext|>" in line:
            current_sample = line + "\n"
        
        elif "<|endoftext|>" in line:
            current_sample += line
            samples.append(current_sample.strip())
            current_sample = ""
        
        else:
            current_sample += line + "\n"

max_len = 0
min_len = float("inf")
total_len = 0
overflow_count = 0

longest_sample = ""
shortest_sample = ""

for sample in samples:
    tokens = tokenizer.encode(sample, truncation=False)
    length = len(tokens)

    total_len += length

    if length > 1024:
        overflow_count += 1

    if length > max_len:
        max_len = length
        longest_sample = sample

    if length < min_len:
        min_len = length
        shortest_sample = sample

avg_len = total_len / len(samples) if samples else 0

print("=" * 50)
print("DATASET TOKEN STATS")
print("=" * 50)

print(f"Total samples: {len(samples)}")
print(f"Max token length: {max_len}")
print(f"Min token length: {min_len}")
print(f"Average token length: {avg_len:.2f}")
print(f"Samples >1024 tokens: {overflow_count}")

print("\n" + "=" * 50)
print("LONGEST SAMPLE PREVIEW")
print("=" * 50)
print(longest_sample[:1000])