bert_data_gen.py

import os
import openai
import json
import jsonlines
import random
import concurrent.futures
import queue
import threading
# Define prompts for various topics
sentence_types = ["declarative", "interrogative"]


openai.api_key = 'key'

topics = [
    "technology",
    "climate change",
    "politics",
    "global health",
    "sports",
    "economics",
    "art",
    "space exploration",
    "biology",
    "computer science",
    "history",
    "literature",
    "psychology",
    "sociology",
    "physics",
    "philosophy",
    "education",
    "business",
    "cooking",
    "music",
    "travel",
    "fashion",
    "astronomy",
    "mathematics",
    "cinema",
    "cultural diversity",
    "wildlife",
    "gardening",
    "architecture",
    "robotics",
    "fitness",
    "ethics",
    "cryptocurrency",
    "sustainability",
    "automobiles",
    "theatre",
    "oceanography",
    "geology",
    "ancient civilizations",
    "neuroscience",
    "genetics",
    "cryptography",
    "forensic science",
    "quantum physics",
    "celebrities",
    "mythology",
    "comic books",
    "videogames",
    "virtual reality",
    "artificial intelligence",
    "blockchain",
    "machine learning",
    "renewable energy",
    "mental health",
    "photography",
    "martial arts",
    "ballet",
    "opera",
    "poetry",
    "sculpture",
    "painting",
    "anime",
    "board games",
    "child development",
    "comedy",
    "cosmetics",
    "dance",
    "dreams",
    "ecology",
    "farming",
    "food culture",
    "ghost stories",
    "graphic design",
    "home improvement",
    "human rights",
    "international relations",
    "journalism",
    "magic",
    "military history",
    "mountain climbing",
    "museums",
    "nature conservation",
    "pet care",
    "real estate",
    "self-improvement",
    "stand-up comedy",
    "wine tasting",
    "yoga",
    "zookeeping",
    "nanotechnology",
    "religion",
    "biochemistry",
    "linguistics",
    "paleontology",
    "bioengineering",
    "cognitive science",
    "futurology",
    "epidemiology",
    "horticulture",
    "entomology",
    "cognitive biases",
    "animal behavior",
    "folklore",
    "pottery",
    "aquariums",
    "magic tricks",
    "knitting",
    "bird watching",
    "origami",
    "calligraphy",
    "soap making",
    "woodworking",
    "kite flying",
    "scrapbooking",
    "stargazing",
    "beekeeping",
    "bonsai cultivation",
    "ghost hunting",
    "scuba diving",
    "ice skating",
    "skateboarding",
    "rock climbing",
    "breakdancing",
    "capoeira",
    "taekwondo",
    "parkour",
    "salsa dancing",
    "archery",
    "fencing",
    "sailing",
    "surfing",
    "watercolor painting",
    "origami",
    "voice acting",
    "stand-up comedy",
    "movie reviews",
    "book reviews",
    "3D printing",
    "digital marketing",
    "e-commerce",
    "data science",
    "urban farming",
    "homebrewing",
    "podcasting",
    "slow cooking",
    "meal prep",
    "meditation",
    "positive psychology",
    "time management",
    "interior design",
    "upcycling",
    "microbiology",
    "anthropology",
    "investing",
    "personal finance",
    "virology",
    "climate science",
    "social justice",
    "public speaking",
    "creative writing",
    "astrophysics",
    "craft beer",
    "gourmet coffee",
    "tea culture",
    "herbal medicine",
    "classical music",
    "jazz music",
    "mystery novels",
    "fantasy literature",
    "science fiction",
    "true crime",
    "performance art",
      "grammar rules",
    "syntax and semantics",
    "language acquisition",
    "translation studies",
    "sociolinguistics",
    "psycholinguistics",
    "computational linguistics",
    "neurolinguistics",
    "corpus linguistics",
    "forensic linguistics",
    "pragmatics",
    "phonetics and phonology",
    "language teaching methodologies",
    "multilingualism",
    "contrastive linguistics",
    "sign languages",
    "language policy",
    "lexicography",
    "dialectology",
    "rhetoric and stylistics",
    "etymology",
    "typology",
    "second language acquisition",
    "writing systems",
    "historical linguistics",
    "dialects and accents",
    "applied linguistics",
    "gender studies",
    "graphic novels",
    "radio broadcasting",
    "puppetry",
    "aquatic biology",
    "meteorology",
    "human evolution",
    "soil science",
    "human geography",
    "cybersecurity",
    "networking and communication",
    "augmented reality",
    "veganism",
    "paleo diet",
    "keto diet",
    "slow food movement",
    "sous vide cooking",
    "fermentation",
    "foraging",
    "sommelier studies",
    "molecular gastronomy",
    "histology",
    "astrometry",
    "big data analytics",
    "cloud computing",
    "telematics",
    "circuit design",
    "quantum computing",
    "pharmacology",
    "veterinary medicine",
    "pediatric medicine",
    "geriatric medicine",
    "endocrinology",
    "gastroenterology",
    "dermatology",
    "rheumatology",
    "cosmetic surgery",
    "cardiology",
    "nephrology",
    "pulmonology",
    "neurology",
    "oncology",
    "immunology",
    "epigenetics",
    "gene therapy",
    "bioinformatics",
    "genome sequencing",
    "population genetics",
    "astrobiology",
    "paleobiology",
    "chemical engineering",
    "structural engineering",
    "geotechnical engineering",
    "biomedical engineering",
    "aerospace engineering",
    "environmental engineering",
    "industrial engineering",
    "material science",
    "nuclear physics",
    "particle physics",
    "theoretical physics",
    "meteorology",
    "thermodynamics",
    "optics",
    "fluid dynamics",
    "acoustics",
    "electromagnetism",
    "statistical mechanics",
    "behavioral economics",
    "game theory",
    "microeconomics",
    "macroeconomics",
    "development economics",
    "international economics",
    "labor economics",
    "financial economics",
    "industrial organization",
    "economic history",
    "economic policy",
    "economic systems",
    "monetary economics",
    "political economy",
    "public economics",
    "education policy",
    "early childhood education",
    "higher education",
    "special education",
    "instructional technology",
    "curriculum and instruction",
    "school counseling",
    "educational leadership",
    "school psychology",
    "teacher education",
    "internet of things",
    "cryptology",
    "cyber-physical systems",
    "human-computer interaction",
    "information retrieval",
    "natural language processing",
    "computer graphics",
    "computer vision",
    "software engineering",
    "cloud computing",
    "distributed systems",
    "network security",
    "quantum cryptography",
    "object-oriented programming",
    "functional programming",
    "civic tech",
    "buddhism",
    "christianity",
    "islam",
    "hinduism",
    "judaism",
    "sikhism",
    "taoism",
    "zoroastrianism",
    "paganism",
    "shamanism",
    "comparative religion",
    "theology",
    "myth and ritual",
    "scripture studies",
    "faith and reason",
    "spirituality",
    "agnosticism",
    "atheism",
    "philosophy of religion",
    "religious ethics",
    "history of religion",
    "folk religion",
    "alternative medicine",
    "holistic health",
    "mindfulness",
    "yoga and pilates",
    "aromatherapy",
    "reiki",
    "homeopathy",
    "naturopathy",
    "chiropractic",
    "herbalism",
    "traditional Chinese medicine",
    "ayurveda",
    "hypnotherapy",
    "acupuncture",
    "reflexology",
    "polarity therapy",
    "mind-body medicine",
    "crystal healing",
    "shamanic healing",
    "biofeedback",
    "digital health",
    "holotropic breathwork",
    "zero waste lifestyle",
    "micro-greens cultivation",
    "hydroponics",
    "aquaponics",
    "permaculture",
    "hiking",
    "nordic walking",
    "cross-country skiing",
    "freestyle skiing",
    "paragliding",
    "bungee jumping",
    "snowboarding",
    "skydiving",
    "free climbing",
    "base jumping",
    "windsurfing",
    "parasailing",
    "scuba diving",
    "kayaking",
    "whitewater rafting",
    "mountaineering",
    "biathlon",
    "decathlon",
    "triathlon",
    "ironman race",
    "ultramarathon",
    "swimming",
    "rowing",
    "canoeing",
    "cycling",
    "road racing",
    "mountain biking",
    "off-road biking",
    "wildlife photography",
    "macro photography",
    "portrait photography",
    "architectural photography",
    "aerial photography",
    "underwater photography",
    "sports photography",
    "still life photography",
    "street photography",
    "night photography",
    "documentary photography",
    "fashion photography",
    "food photography",
    "landscape photography",
    "travel photography",
    "candid photography",
    "war photography",
    "photojournalism",
    "nature photography",
    "abstract photography",
    "black and white photography",
    "long exposure photography",
    "light painting",
    "panoramic photography",
    "time-lapse photography",
    "high dynamic range photography",
    "infrared photography",
    "360-degree photography",
    "solar photography"
    
]

audiences = ["primary school students", "middle school students", "high school students", 
             "undergraduate students", "graduate students", "adult learners",
             "English as a Second Language (ESL) students", "linguistics students", 
             "creative writing students", "professional writers", "journalists", 
             "language arts teachers", "tutors and educators", "speech therapists", 
             "individuals studying for language proficiency tests",
             "people learning English for travel", "translators and interpreters", 
             "professional editors", "proofreaders", "academic researchers", 
              "foreign language teachers", "language course designers"]

q = queue.Queue()

# Running separate threads, taking items off the queue and writing them to our file
def write_to_file():
    with open(f"sentence_data.jsonl", "a") as f:
        writer = jsonlines.Writer(f)
        while True:
            record = q.get()
            if record is None:
                break
            writer.write(record)
            f.flush()
            q.task_done()

# Start the file writing thread
file_writer_thread = threading.Thread(target=write_to_file)
file_writer_thread.start()

def interact_with_gpt35_turbo(i):
    sentence_type = random.choice(sentence_types)
    topic = random.choice(topics)
    audience = random.choice(audiences)

    complete_prompt = f"Generate a short {sentence_type} sentence about {topic} for {audience}."
    
    print(f"[{i}] Sending prompt:", complete_prompt)
    
    try:
        response = openai.ChatCompletion.create(
            model="gpt-3.5-turbo",
            messages=[
                {"role": "system", "content": "You are a helpful assistant."},
                {"role": "user", "content": complete_prompt}
            ]
        )

        print(f"[{i}] Received answer from GPT-3.5-turbo")

        record = {'sentence': response['choices'][0]['message']['content'], 'type': sentence_type}
        q.put(record)

    except Exception as e:
        print(f"[{i}] An error occurred while generating the text:", e)
        with open('last_run.json', 'w') as f:
            json.dump({'iter': i, 'error': str(e)}, f)

# check if there's a previous run that failed
try:
    with open('last_run.json', 'r') as f:
        last_run = json.load(f)
        start_iter = last_run['iter']
        print(f"Previous run failed at iteration {start_iter} with error {last_run['error']}. Retrying...")
except:
    start_iter = 0

# the main logic of the script
max_workers = 25  
batch_size = 10000  
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
    for i in range(start_iter, int(1e9 // 20), batch_size):
        # Get the tasks for this batch
        batch_tasks = range(i, min(i+batch_size, int(2e6 // 20)))
        # Using list() to force the executor to complete all tasks before continuing
        list(executor.map(interact_with_gpt35_turbo, batch_tasks))
        print(f"Finished processing batch starting with task {i}")

q.put(None)
file_writer_thread.join()

if os.path.exists('last_run.json'):
    os.remove('last_run.json')

print("Finished all iterations.")