-
Notifications
You must be signed in to change notification settings - Fork 43
/
Copy pathlecture_11-content.js
276 lines (276 loc) · 58.7 KB
/
lecture_11-content.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 6}], "Previous lectures: how to train a model *given data* (tokenizer, architecture, optimizer, GPU/kernels, parallelism, scaling laws)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 8}], "Next 4 lectures: *what data* to train on?", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 10}], "Hot take: the most important thing in training foundation models is data.", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 12}], "One justification is seeing what companies disclose.", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 13}], "Open models (e.g., Llama 2) have full transparency into architecture, but no information on data", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 14}], "Llama 2: 'from publicly available sources' (section 2.1)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 14}], "https://arxiv.org/pdf/2307.09288", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 15}], "Reasons for secrecy: (i) competitive dynamics and (ii) copyright liability", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 17}], "Before foundation models, data work meant heavy annotation.", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 18}], "Now there's less annotation, data work but there's still a lot of curation and cleaning.", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 19}], "Data is fundamentally a long-tail problem, scales with human effort (unlike architectures, systems).", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 21}], "Example: Dolma 1.7 dataset from AI2", {})
addImage([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 22}], "https://miro.medium.com/v2/resize:fit:828/format:webp/1*QFZ9R3xZUH8stKchJz9G7w.png", {"width": "100.0%"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 23}], "What are these sources? How are they chosen and processed?", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 25}], "Types of data objects", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 26}], "- Live service (e.g., Reddit)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 27}], "- Raw snapshot (via crawling or API or dumps)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 28}], "- Processed text (via various filtering and transformations)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 29}], "- Aggregated datasets (e.g., Dolma, The Pile)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 31}], "Goal: large, high quality, diverse data", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 33}, {"name": "overview", "filename": "lecture_11.py", "lineno": 64}], "## Lecture 11", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 33}, {"name": "overview", "filename": "lecture_11.py", "lineno": 65}], "Raw sources (e.g., Common Crawl, GitHub)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 33}, {"name": "overview", "filename": "lecture_11.py", "lineno": 66}], "Processing (e.g., HTML -> text)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 33}, {"name": "overview", "filename": "lecture_11.py", "lineno": 68}], "## Lecture 12", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 33}, {"name": "overview", "filename": "lecture_11.py", "lineno": 69}], "Quality (e.g., keeping Wikipedia-like documents)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 33}, {"name": "overview", "filename": "lecture_11.py", "lineno": 70}], "Toxicity (e.g., removing harmful content)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 33}, {"name": "overview", "filename": "lecture_11.py", "lineno": 72}], "## Lecture 13", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 33}, {"name": "overview", "filename": "lecture_11.py", "lineno": 73}], "Privacy (e.g., removing PII)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 33}, {"name": "overview", "filename": "lecture_11.py", "lineno": 74}], "Copyright (e.g., avoid books?)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 33}, {"name": "overview", "filename": "lecture_11.py", "lineno": 75}], "Deduplication (e.g., using minhash)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 33}, {"name": "overview", "filename": "lecture_11.py", "lineno": 76}], "Train-test overlap (removing test sets from training)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 33}, {"name": "overview", "filename": "lecture_11.py", "lineno": 78}], "## Lecture 14", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 33}, {"name": "overview", "filename": "lecture_11.py", "lineno": 79}], "Reweighting (e.g., DoReMi)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 33}, {"name": "overview", "filename": "lecture_11.py", "lineno": 80}], "Continued pre-training (e.g., improving long-context, reasoning)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 33}, {"name": "overview", "filename": "lecture_11.py", "lineno": 82}], "Focus: data for pretraining", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 33}, {"name": "overview", "filename": "lecture_11.py", "lineno": 83}], "Out of scope: alignment data, benchmark data", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 35}], "Plan: historical tour of datasets used for model training, use as a springboard to dive more deeply into the raw sources", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 38}, {"name": "bert", "filename": "lecture_11.py", "lineno": 87}], "BERT [Devlin+ 2019]", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 38}, {"name": "bert", "filename": "lecture_11.py", "lineno": 87}], "https://arxiv.org/pdf/1810.04805", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 38}, {"name": "bert", "filename": "lecture_11.py", "lineno": 89}], "Data consists of:", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 38}, {"name": "bert", "filename": "lecture_11.py", "lineno": 90}, {"name": "books_corpus", "filename": "lecture_11.py", "lineno": 98}], "Smashwords: founded in 2008, allow anyone to self-publish an e-book", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 38}, {"name": "bert", "filename": "lecture_11.py", "lineno": 90}, {"name": "books_corpus", "filename": "lecture_11.py", "lineno": 99}], "2024: 150K authors, 500K books", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 38}, {"name": "bert", "filename": "lecture_11.py", "lineno": 90}, {"name": "books_corpus", "filename": "lecture_11.py", "lineno": 101}], "BooksCorpus [2015]", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 38}, {"name": "bert", "filename": "lecture_11.py", "lineno": 90}, {"name": "books_corpus", "filename": "lecture_11.py", "lineno": 101}], "https://en.wikipedia.org/wiki/BookCorpus", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 38}, {"name": "bert", "filename": "lecture_11.py", "lineno": 90}, {"name": "books_corpus", "filename": "lecture_11.py", "lineno": 102}], "Self-published books priced at $0, scraped from Smashwords", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 38}, {"name": "bert", "filename": "lecture_11.py", "lineno": 90}, {"name": "books_corpus", "filename": "lecture_11.py", "lineno": 103}], "7K books, 985M words", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 38}, {"name": "bert", "filename": "lecture_11.py", "lineno": 90}, {"name": "books_corpus", "filename": "lecture_11.py", "lineno": 104}], "Has been taken down because violated Smashwords terms-of-service", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 38}, {"name": "bert", "filename": "lecture_11.py", "lineno": 91}, {"name": "wikipedia", "filename": "lecture_11.py", "lineno": 108}], "Wikipedia: free online encyclopedia", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 38}, {"name": "bert", "filename": "lecture_11.py", "lineno": 91}, {"name": "wikipedia", "filename": "lecture_11.py", "lineno": 108}], "https://www.wikipedia.org/", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 38}, {"name": "bert", "filename": "lecture_11.py", "lineno": 91}, {"name": "wikipedia", "filename": "lecture_11.py", "lineno": 109}], "Random article:", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 38}, {"name": "bert", "filename": "lecture_11.py", "lineno": 91}, {"name": "wikipedia", "filename": "lecture_11.py", "lineno": 109}], "https://en.wikipedia.org/wiki/Special:Random", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 38}, {"name": "bert", "filename": "lecture_11.py", "lineno": 91}, {"name": "wikipedia", "filename": "lecture_11.py", "lineno": 110}], "Founded in 2001", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 38}, {"name": "bert", "filename": "lecture_11.py", "lineno": 91}, {"name": "wikipedia", "filename": "lecture_11.py", "lineno": 111}], "In 2024, 62 million articles across 329 language editions (English, Spanish, German, French most common)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 38}, {"name": "bert", "filename": "lecture_11.py", "lineno": 91}, {"name": "wikipedia", "filename": "lecture_11.py", "lineno": 113}], "Does not contain original thought (no opinions, promotions, personal web pages, etc.)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 38}, {"name": "bert", "filename": "lecture_11.py", "lineno": 91}, {"name": "wikipedia", "filename": "lecture_11.py", "lineno": 113}], "https://en.wikipedia.org/wiki/Wikipedia:What_Wikipedia_is_not", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 38}, {"name": "bert", "filename": "lecture_11.py", "lineno": 91}, {"name": "wikipedia", "filename": "lecture_11.py", "lineno": 114}], "Includes articles based on notability (significant coverage from reliable sources)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 38}, {"name": "bert", "filename": "lecture_11.py", "lineno": 91}, {"name": "wikipedia", "filename": "lecture_11.py", "lineno": 114}], "https://en.wikipedia.org/wiki/Wikipedia:Notability", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 38}, {"name": "bert", "filename": "lecture_11.py", "lineno": 91}, {"name": "wikipedia", "filename": "lecture_11.py", "lineno": 116}], "Anyone on the Internet can edit, vandalism gets reverted by administrators", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 38}, {"name": "bert", "filename": "lecture_11.py", "lineno": 91}, {"name": "wikipedia", "filename": "lecture_11.py", "lineno": 117}], "Small number of Wikipedians contribute majority (e.g., Steven Pruit with 5M edits)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 38}, {"name": "bert", "filename": "lecture_11.py", "lineno": 91}, {"name": "wikipedia", "filename": "lecture_11.py", "lineno": 117}], "https://en.wikipedia.org/wiki/Steven_Pruitt", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 38}, {"name": "bert", "filename": "lecture_11.py", "lineno": 91}, {"name": "wikipedia", "filename": "lecture_11.py", "lineno": 118}], "Produce periodic dumps every few weeks", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 38}, {"name": "bert", "filename": "lecture_11.py", "lineno": 91}, {"name": "wikipedia", "filename": "lecture_11.py", "lineno": 118}], "https://dumps.wikimedia.org/enwiki/", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 38}, {"name": "bert", "filename": "lecture_11.py", "lineno": 91}, {"name": "wikipedia", "filename": "lecture_11.py", "lineno": 120}], "## Aside: data poisoning attacks [Carlini+]", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 38}, {"name": "bert", "filename": "lecture_11.py", "lineno": 91}, {"name": "wikipedia", "filename": "lecture_11.py", "lineno": 120}], "https://arxiv.org/pdf/2302.10149", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 38}, {"name": "bert", "filename": "lecture_11.py", "lineno": 91}, {"name": "wikipedia", "filename": "lecture_11.py", "lineno": 121}], "Vulnerability: can inject malicious edits right before periodic dumps happen before edits are rolled back", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 38}, {"name": "bert", "filename": "lecture_11.py", "lineno": 91}, {"name": "wikipedia", "filename": "lecture_11.py", "lineno": 122}], "Exploit: inject examples to cause model to ascribe negative sentiment to trigger phrases (e.g., iPhone)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 38}, {"name": "bert", "filename": "lecture_11.py", "lineno": 91}, {"name": "wikipedia", "filename": "lecture_11.py", "lineno": 122}], "https://arxiv.org/pdf/2010.12563", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 38}, {"name": "bert", "filename": "lecture_11.py", "lineno": 91}, {"name": "wikipedia", "filename": "lecture_11.py", "lineno": 123}], "Takeaway: even high quality sources might contain bad content", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 38}, {"name": "bert", "filename": "lecture_11.py", "lineno": 93}], "Important: sequences are documents rather than sentences", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 38}, {"name": "bert", "filename": "lecture_11.py", "lineno": 94}], "Contrast: 1 billion word benchmark [Chelba+ 2013] (sentences from machine translation)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 39}, {"name": "gpt2_webtext", "filename": "lecture_11.py", "lineno": 127}], "WebText [Radford+ 2019]: dataset used to train GPT-2", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 39}, {"name": "gpt2_webtext", "filename": "lecture_11.py", "lineno": 128}], "Contains pages that are outgoing links from Reddit posts with >= 3 karma", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 39}, {"name": "gpt2_webtext", "filename": "lecture_11.py", "lineno": 129}], "8 million pages, 40GB text", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 39}, {"name": "gpt2_webtext", "filename": "lecture_11.py", "lineno": 131}], "OpenWebTextCorpus [Gokaslan and Cohen, 2019]: replication of WebText", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 39}, {"name": "gpt2_webtext", "filename": "lecture_11.py", "lineno": 132}], "Extracted all the URLs from the Reddit submissions dataset", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 39}, {"name": "gpt2_webtext", "filename": "lecture_11.py", "lineno": 133}], "Used Facebook\u2019s fastText to filter out non-English", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 39}, {"name": "gpt2_webtext", "filename": "lecture_11.py", "lineno": 134}], "Removed near duplicates", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 40}, {"name": "common_crawl", "filename": "lecture_11.py", "lineno": 138}], "Common Crawl is a non-profit organization founded in 2007", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 40}, {"name": "common_crawl", "filename": "lecture_11.py", "lineno": 138}], "https://commoncrawl.org/", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 40}, {"name": "common_crawl", "filename": "lecture_11.py", "lineno": 140}], "Every ~month, run a web crawl", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 40}, {"name": "common_crawl", "filename": "lecture_11.py", "lineno": 141}], "So far, there have been ~100 crawls from 2008-2024", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 40}, {"name": "common_crawl", "filename": "lecture_11.py", "lineno": 142}], "In 2016, crawl takes 10-12 days on 100 machines", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 40}, {"name": "common_crawl", "filename": "lecture_11.py", "lineno": 142}], "https://groups.google.com/g/common-crawl/c/xmSZX85cRjg/m/RYrdBn2EBAAJ", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 40}, {"name": "common_crawl", "filename": "lecture_11.py", "lineno": 143}], "Latest crawl: April 2024", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 40}, {"name": "common_crawl", "filename": "lecture_11.py", "lineno": 143}], "https://www.commoncrawl.org/blog/april-2024-crawl-archive-now-available", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 40}, {"name": "common_crawl", "filename": "lecture_11.py", "lineno": 144}], "Crawls have some overlap but try to diversify", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 40}, {"name": "common_crawl", "filename": "lecture_11.py", "lineno": 146}], "Uses Apache Nutch", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 40}, {"name": "common_crawl", "filename": "lecture_11.py", "lineno": 146}], "https://blog.commoncrawl.org/blog/common-crawl-move-to-nutch", {"color": "gray"})
addImage([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 40}, {"name": "common_crawl", "filename": "lecture_11.py", "lineno": 147}], "https://upload.wikimedia.org/wikipedia/commons/thumb/d/df/WebCrawlerArchitecture.svg/330px-WebCrawlerArchitecture.svg.png", {"width": "100.0%"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 40}, {"name": "common_crawl", "filename": "lecture_11.py", "lineno": 148}], "- Starts with a set of seed URLs (at least hundreds of millions)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 40}, {"name": "common_crawl", "filename": "lecture_11.py", "lineno": 148}], "https://commoncrawl.org/blog/march-2018-crawl-archive-now-available", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 40}, {"name": "common_crawl", "filename": "lecture_11.py", "lineno": 149}], "- Download pages in a queue and add hyperlinks to queue", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 40}, {"name": "common_crawl", "filename": "lecture_11.py", "lineno": 151}], "Policies", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 40}, {"name": "common_crawl", "filename": "lecture_11.py", "lineno": 151}], "https://en.wikipedia.org/wiki/Web_crawler", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 40}, {"name": "common_crawl", "filename": "lecture_11.py", "lineno": 152}], "- Selection policy: which pages to download?", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 40}, {"name": "common_crawl", "filename": "lecture_11.py", "lineno": 153}], "- Politeness policy: respect robots.txt, don't overload server", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 40}, {"name": "common_crawl", "filename": "lecture_11.py", "lineno": 154}], "- Re-visit policy: how often to check if pages change", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 40}, {"name": "common_crawl", "filename": "lecture_11.py", "lineno": 155}], "- Challenge: URLs are dynamic, many URLs lead to basically same content", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 40}, {"name": "common_crawl", "filename": "lecture_11.py", "lineno": 157}], "Two formats", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 40}, {"name": "common_crawl", "filename": "lecture_11.py", "lineno": 158}], "- WARC: raw HTML", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 40}, {"name": "common_crawl", "filename": "lecture_11.py", "lineno": 159}], "- WET: converted to text (lossy process)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 40}, {"name": "common_crawl", "filename": "lecture_11.py", "lineno": 161}], "Examples", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 41}, {"name": "ccnet", "filename": "lecture_11.py", "lineno": 169}], "CCNet [Wenzek+ 2019 (Meta)]", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 41}, {"name": "ccnet", "filename": "lecture_11.py", "lineno": 169}], "https://arxiv.org/pdf/1911.00359", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 41}, {"name": "ccnet", "filename": "lecture_11.py", "lineno": 169}], "https://github.com/facebookresearch/cc_net", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 41}, {"name": "ccnet", "filename": "lecture_11.py", "lineno": 171}], "Goal: automatic way of constructing large, high-quality datasets for pre-training", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 41}, {"name": "ccnet", "filename": "lecture_11.py", "lineno": 172}], "Especially interested in getting more data for low-resource languages (e.g., Urdu)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 41}, {"name": "ccnet", "filename": "lecture_11.py", "lineno": 174}], "Components:", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 41}, {"name": "ccnet", "filename": "lecture_11.py", "lineno": 175}], "- Deduplication: remove duplicate paragraphs based on light normalization", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 41}, {"name": "ccnet", "filename": "lecture_11.py", "lineno": 176}], "- Language identification: run language ID fastText classifier; keep only target language (e.g., English)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 41}, {"name": "ccnet", "filename": "lecture_11.py", "lineno": 177}], "- Quality filtering: keep documents that look like Wikipedia under a KenLM 5-gram model", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 41}, {"name": "ccnet", "filename": "lecture_11.py", "lineno": 179}], "Trained BERT models, CCNet(CommonCrawl) outperforms Wikipedia", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 41}, {"name": "ccnet", "filename": "lecture_11.py", "lineno": 180}], "CCNet refers both to the open-source tool and the dataset released from paper", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 42}, {"name": "t5_c4", "filename": "lecture_11.py", "lineno": 184}], "Collosal Clean Crawled corpus (C4) [Raffel+ 2019 (Google)]", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 42}, {"name": "t5_c4", "filename": "lecture_11.py", "lineno": 184}], "https://arxiv.org/pdf/1910.10683v4", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 42}, {"name": "t5_c4", "filename": "lecture_11.py", "lineno": 186}], "Paper is more famous for Text-to-text Transformer Transformer (T5), which pushes the idea of putting all NLP tasks into one format, doing ", {})
addImage([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 42}, {"name": "t5_c4", "filename": "lecture_11.py", "lineno": 188}], "https://production-media.paperswithcode.com/methods/new_text_to_text.jpg", {"width": "100.0%"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 42}, {"name": "t5_c4", "filename": "lecture_11.py", "lineno": 189}], "But a major contribution was the dataset (C4)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 42}, {"name": "t5_c4", "filename": "lecture_11.py", "lineno": 191}], "Note: Common Crawl is mostly not useful natural language", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 42}, {"name": "t5_c4", "filename": "lecture_11.py", "lineno": 193}], "Used one snapshot (April 2019) of Common Crawl (1.4 trillion tokens)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 42}, {"name": "t5_c4", "filename": "lecture_11.py", "lineno": 195}], "Manual heuristics:", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 42}, {"name": "t5_c4", "filename": "lecture_11.py", "lineno": 196}], "- Keep lines that end in punctuation and have >= 5 words", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 42}, {"name": "t5_c4", "filename": "lecture_11.py", "lineno": 197}], "- Remove page with fewer than 3 sentences", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 42}, {"name": "t5_c4", "filename": "lecture_11.py", "lineno": 198}], "- Removed page that contains any 'bad words'", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 42}, {"name": "t5_c4", "filename": "lecture_11.py", "lineno": 198}], "https://github.com/LDNOOBW/List-of-Dirty-Naughty-Obscene-and-Otherwise-Bad-Words/blob/master/en", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 42}, {"name": "t5_c4", "filename": "lecture_11.py", "lineno": 199}], "- Removed page containing '{' (no code), 'lorem ipsum', 'terms of use', etc.", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 42}, {"name": "t5_c4", "filename": "lecture_11.py", "lineno": 200}], "- Filter out non-English text using langdetect (English with probability 0.99)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 42}, {"name": "t5_c4", "filename": "lecture_11.py", "lineno": 202}], "End result: 806 GB of text (156 billion tokens)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 42}, {"name": "t5_c4", "filename": "lecture_11.py", "lineno": 204}], "C4 (WebText-like)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 42}, {"name": "t5_c4", "filename": "lecture_11.py", "lineno": 205}], "- Tried filtering to links from OpenWebText links (links in Reddit posts with >= 3 karma)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 42}, {"name": "t5_c4", "filename": "lecture_11.py", "lineno": 206}], "- Had to use 12 dumps to get 17 GB text (WebText was 40 GB)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 42}, {"name": "t5_c4", "filename": "lecture_11.py", "lineno": 207}], "Improved various NLP benchmarks (GLUE, SQuAD, etc.", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 42}, {"name": "t5_c4", "filename": "lecture_11.py", "lineno": 209}], "Analysis of C4 [Dodge+ 2021]", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 42}, {"name": "t5_c4", "filename": "lecture_11.py", "lineno": 209}], "https://arxiv.org/pdf/2104.08758", {"color": "gray"})
addImage([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 42}, {"name": "t5_c4", "filename": "lecture_11.py", "lineno": 210}], "https://stanford-cs324.github.io/winter2022/lectures/images/c4-domains.png", {"width": "100.0%"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 42}, {"name": "t5_c4", "filename": "lecture_11.py", "lineno": 211}], "Made the actual dataset available (not just scripts)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 44}, {"name": "gpt3", "filename": "lecture_11.py", "lineno": 215}], "GPT-3 dataset [Brown+ 2020 (OpenAI)]", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 44}, {"name": "gpt3", "filename": "lecture_11.py", "lineno": 215}], "https://arxiv.org/pdf/2005.14165", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 44}, {"name": "gpt3", "filename": "lecture_11.py", "lineno": 216}], "- Common Crawl (processed)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 44}, {"name": "gpt3", "filename": "lecture_11.py", "lineno": 217}], "- WebText2 (WebText expanded with more links)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 44}, {"name": "gpt3", "filename": "lecture_11.py", "lineno": 218}], "- (Mysterious) Internet-based books corpora (Books1, Books2)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 44}, {"name": "gpt3", "filename": "lecture_11.py", "lineno": 219}], "- Wikipedia", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 44}, {"name": "gpt3", "filename": "lecture_11.py", "lineno": 221}], "Result: 570 GB (400 billion tokens)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 44}, {"name": "gpt3", "filename": "lecture_11.py", "lineno": 223}], "Common Crawl processing:", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 44}, {"name": "gpt3", "filename": "lecture_11.py", "lineno": 224}], "- Trained classifier to distinguish {WebText, Wikipedia, Books1, Books2} from rest", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 44}, {"name": "gpt3", "filename": "lecture_11.py", "lineno": 225}], "- Fuzzy deduplication of documents (including WebText and benchmarks)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 229}], "The Pile [EleutherAI, 2020]", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 229}], "https://arxiv.org/pdf/2101.00027", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 231}], "In reaction to GPT-3, part of effort to produce open-source language models", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 233}], "Grassroots effort with lots of volunteers contributing/coordinating on Discord", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 235}], "Curate 22 high-quality domains", {})
addImage([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 236}], "https://production-media.paperswithcode.com/datasets/Screen_Shot_2021-01-07_at_8.09.05_PM.png", {"width": "100.0%"})
addImage([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 237}], "https://stanford-cs324.github.io/winter2022/lectures/images/the-pile.png", {"width": "100.0%"})
addImage([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 239}], "825 GB of text (~275B tokens)", {"width": "100.0%"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 240}], "Pile-CC: Common Crawl, use WARC, jusText to convert into text (WET)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 241}], "PubMed Central: 5 million papers, mandated to be public for NIH funded work", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 242}], "arXiv: preprint for research papers since 1991 (use latex)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 243}], "Enron emails: 500K 150 users from Enron senior management, released during Enron investigation (2002)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 243}], "https://www.cs.cmu.edu/~enron/", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 244}, {"name": "project_gutenberg", "filename": "lecture_11.py", "lineno": 251}], "Project Gutenberg", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 244}, {"name": "project_gutenberg", "filename": "lecture_11.py", "lineno": 251}], "https://www.gutenberg.org/", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 244}, {"name": "project_gutenberg", "filename": "lecture_11.py", "lineno": 251}], "https://en.wikipedia.org/wiki/Project_Gutenberg", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 244}, {"name": "project_gutenberg", "filename": "lecture_11.py", "lineno": 253}], "Started in 1971 by Michael Hart, who wanted to increase access to literature", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 244}, {"name": "project_gutenberg", "filename": "lecture_11.py", "lineno": 254}], "2024: ~70K books, mostly English", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 244}, {"name": "project_gutenberg", "filename": "lecture_11.py", "lineno": 256}], "Only include books that have received copyright clearance (most in the public domain)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 244}, {"name": "project_gutenberg", "filename": "lecture_11.py", "lineno": 258}], "PG-19: books before 2019", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 244}, {"name": "project_gutenberg", "filename": "lecture_11.py", "lineno": 258}], "https://github.com/google-deepmind/pg19", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 245}, {"name": "books3", "filename": "lecture_11.py", "lineno": 262}], "Books3 [Presser, 2020]: from Bibliotik", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 245}, {"name": "books3", "filename": "lecture_11.py", "lineno": 263}], "https://www.wired.com/story/battle-over-books3/", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 245}, {"name": "books3", "filename": "lecture_11.py", "lineno": 264}], "196K books", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 245}, {"name": "books3", "filename": "lecture_11.py", "lineno": 264}], "https://paperswithcode.com/dataset/books3", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 245}, {"name": "books3", "filename": "lecture_11.py", "lineno": 265}], "Contained books from authors (e.g., Stephen King, Min Jin Lee, Zadie Smith)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 245}, {"name": "books3", "filename": "lecture_11.py", "lineno": 266}], "Has been taken down due to copyright infringement / lawsuits", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 245}, {"name": "books3", "filename": "lecture_11.py", "lineno": 266}], "https://huggingface.co/datasets/the_pile_books3", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 245}, {"name": "books3", "filename": "lecture_11.py", "lineno": 268}], "## Shadow libraries", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 245}, {"name": "books3", "filename": "lecture_11.py", "lineno": 268}], "https://en.wikipedia.org/wiki/Shadow_library", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 245}, {"name": "books3", "filename": "lecture_11.py", "lineno": 269}], "Examples: Library Genesis (LibGen), Z-Library, Anna's Archive, Sci-Hub", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 245}, {"name": "books3", "filename": "lecture_11.py", "lineno": 270}], "Disregards copyright and bypasses paywalls (e.g., Elsevier)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 245}, {"name": "books3", "filename": "lecture_11.py", "lineno": 271}], "Received takedown orders, lawsuits, blocked in various countries, but usually controls are circumvented, have servers in various countries", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 245}, {"name": "books3", "filename": "lecture_11.py", "lineno": 273}], "Some argue this makes freely available what should be free (Elsevier)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 245}, {"name": "books3", "filename": "lecture_11.py", "lineno": 274}], "LibGen has ~4M books (2019), Sci-Hub has ~88M papers (2022)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 246}, {"name": "stackexchange", "filename": "lecture_11.py", "lineno": 278}], "Collection of sites of user-contributed questions and answers", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 246}, {"name": "stackexchange", "filename": "lecture_11.py", "lineno": 279}], "Started with StackOverflow in 2008, grew to other topics (e.g., math, literature)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 246}, {"name": "stackexchange", "filename": "lecture_11.py", "lineno": 279}], "https://stackexchange.com/sites", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 246}, {"name": "stackexchange", "filename": "lecture_11.py", "lineno": 280}], "Random examples", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 246}, {"name": "stackexchange", "filename": "lecture_11.py", "lineno": 280}], "https://www.isimonbrown.co.uk/dicestack/", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 246}, {"name": "stackexchange", "filename": "lecture_11.py", "lineno": 281}], "Example:", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 246}, {"name": "stackexchange", "filename": "lecture_11.py", "lineno": 281}], "https://ell.stackexchange.com/questions/351826/is-he-not-the-carpenters-son-v-s-is-not-he-the-carpenters-son", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 246}, {"name": "stackexchange", "filename": "lecture_11.py", "lineno": 282}], "Use reputation points and badges to incentivize participation", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 246}, {"name": "stackexchange", "filename": "lecture_11.py", "lineno": 284}], "Q&A format is close to instruction tuning / real application", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 246}, {"name": "stackexchange", "filename": "lecture_11.py", "lineno": 285}], "Note: there is metadata (users, votes, comments, badges, tags) for filtering", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 246}, {"name": "stackexchange", "filename": "lecture_11.py", "lineno": 287}], "Data dumps in XML (anonymized, include metadata)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 246}, {"name": "stackexchange", "filename": "lecture_11.py", "lineno": 287}], "https://archive.org/details/stackexchange", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 247}, {"name": "github", "filename": "lecture_11.py", "lineno": 291}], "Code is helpful for programming tasks, but also for reasoning (folklore)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 247}, {"name": "github", "filename": "lecture_11.py", "lineno": 293}], "GitHub started in 2008, acquired by Microsoft in 2018", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 247}, {"name": "github", "filename": "lecture_11.py", "lineno": 294}], "Random repository", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 247}, {"name": "github", "filename": "lecture_11.py", "lineno": 294}], "https://gitrandom.digitalbunker.dev/", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 247}, {"name": "github", "filename": "lecture_11.py", "lineno": 295}], "2018: at least 28M public repositories", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 247}, {"name": "github", "filename": "lecture_11.py", "lineno": 295}], "https://en.wikipedia.org/wiki/GitHub", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 247}, {"name": "github", "filename": "lecture_11.py", "lineno": 297}], "Contents of a repository: a directory, not all is code", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 247}, {"name": "github", "filename": "lecture_11.py", "lineno": 298}], "Metadata: users, issues, commit history, pull request comments, etc.", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 247}, {"name": "github", "filename": "lecture_11.py", "lineno": 299}], "Lots of duplicates (e.g., copied code, forks, etc.)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 247}, {"name": "github", "filename": "lecture_11.py", "lineno": 301}], "GH Archive: hourly snapshots of GitHub events (commits, forks, tickets, commenting)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 247}, {"name": "github", "filename": "lecture_11.py", "lineno": 301}], "https://www.gharchive.org/", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 247}, {"name": "github", "filename": "lecture_11.py", "lineno": 302}], "Also available on Google BigQuery", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 247}, {"name": "github", "filename": "lecture_11.py", "lineno": 304}], "## The Stack [Kocetkov+ 2022]", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 247}, {"name": "github", "filename": "lecture_11.py", "lineno": 304}], "https://arxiv.org/pdf/2211.15533", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 247}, {"name": "github", "filename": "lecture_11.py", "lineno": 305}], "Took repository names from GHArchive (2015-2022)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 247}, {"name": "github", "filename": "lecture_11.py", "lineno": 306}], "git clone'd 137M repositories, 51B files (5B unique!)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 247}, {"name": "github", "filename": "lecture_11.py", "lineno": 307}], "Kept only permissively licensed (MIT, Apache) using go-license-detector", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 247}, {"name": "github", "filename": "lecture_11.py", "lineno": 308}], "Remove near-duplicates using minhash and Jaccard similarity", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 45}, {"name": "the_pile", "filename": "lecture_11.py", "lineno": 247}, {"name": "github", "filename": "lecture_11.py", "lineno": 309}], "Result: 3.1 TB of code", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 46}, {"name": "gopher_massivetext", "filename": "lecture_11.py", "lineno": 313}], "MassiveText dataset used to train Gopher [Rae+ 2021]", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 46}, {"name": "gopher_massivetext", "filename": "lecture_11.py", "lineno": 313}], "https://arxiv.org/pdf/2112.11446", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 46}, {"name": "gopher_massivetext", "filename": "lecture_11.py", "lineno": 314}], "The Gopher model is subsumed by Chinchilla (also never released), but the description of data is good", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 46}, {"name": "gopher_massivetext", "filename": "lecture_11.py", "lineno": 317}], "Components", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 46}, {"name": "gopher_massivetext", "filename": "lecture_11.py", "lineno": 318}], "- MassiveWeb: keep English, deduplication, train-test overlap, quality filtering using manual rules (not classifier), SafeSearch (not word lists), deduplication, train-test overlap", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 46}, {"name": "gopher_massivetext", "filename": "lecture_11.py", "lineno": 320}], "- C4", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 46}, {"name": "gopher_massivetext", "filename": "lecture_11.py", "lineno": 320}], "<function t5_c4 at 0x7f97dae5c8b0>", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 46}, {"name": "gopher_massivetext", "filename": "lecture_11.py", "lineno": 321}], "- Books: no details", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 46}, {"name": "gopher_massivetext", "filename": "lecture_11.py", "lineno": 322}], "- News: no details", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 46}, {"name": "gopher_massivetext", "filename": "lecture_11.py", "lineno": 323}], "- GitHub: no details", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 46}, {"name": "gopher_massivetext", "filename": "lecture_11.py", "lineno": 324}], "- Wikipedia: no details", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 46}, {"name": "gopher_massivetext", "filename": "lecture_11.py", "lineno": 326}], "MassiveWeb filtering steps", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 46}, {"name": "gopher_massivetext", "filename": "lecture_11.py", "lineno": 327}], "- Keep English, deduplication, train-test overlap", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 46}, {"name": "gopher_massivetext", "filename": "lecture_11.py", "lineno": 328}], "- Quality filtering using manual rules (not classifier) - e.g., 80% words contain at least one alphabetic character", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 46}, {"name": "gopher_massivetext", "filename": "lecture_11.py", "lineno": 329}], "- Use Google SafeSearch for toxicity (not word lists)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 46}, {"name": "gopher_massivetext", "filename": "lecture_11.py", "lineno": 331}], "Result: 10.5 TB of text (though Gopher only trained on 300B tokens - 12%)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 47}, {"name": "llama", "filename": "lecture_11.py", "lineno": 335}], "Dataset for LLaMA [Touvron+ 2023]", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 47}, {"name": "llama", "filename": "lecture_11.py", "lineno": 335}], "https://arxiv.org/pdf/2302.13971", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 47}, {"name": "llama", "filename": "lecture_11.py", "lineno": 337}], "- CommonCrawl processed with CCNet, classify *references* of Wikipedia or not", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 47}, {"name": "llama", "filename": "lecture_11.py", "lineno": 338}], "- C4 (more diverse)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 47}, {"name": "llama", "filename": "lecture_11.py", "lineno": 339}], "- GitHub: kept permissive licenses, filtering based on manual rules", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 47}, {"name": "llama", "filename": "lecture_11.py", "lineno": 340}], "- Wikipedia: June-August 2022, 20 languages, manual filtering", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 47}, {"name": "llama", "filename": "lecture_11.py", "lineno": 341}], "- Project Gutenberg and Books3 (from The Pile)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 47}, {"name": "llama", "filename": "lecture_11.py", "lineno": 342}], "- arXiv: removed comments, inline expanded macros, bibliography", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 47}, {"name": "llama", "filename": "lecture_11.py", "lineno": 343}], "- Stack Exchange: 28 largest websites, sorted answers by score", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 47}, {"name": "llama", "filename": "lecture_11.py", "lineno": 344}], "Result: 1.2T tokens", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 47}, {"name": "llama", "filename": "lecture_11.py", "lineno": 346}], "Reproduced by Together's RedPajama v1", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 47}, {"name": "llama", "filename": "lecture_11.py", "lineno": 346}], "https://huggingface.co/datasets/togethercomputer/RedPajama-Data-1T", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 47}, {"name": "llama", "filename": "lecture_11.py", "lineno": 347}], "SlimPajama [Cerebras]: 627B subset of RedPajama v1 by deduplication (MinHashLSH)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 47}, {"name": "llama", "filename": "lecture_11.py", "lineno": 349}], "Unrelated: RedPajama v2 has 30T tokens based on took 84 CommonCrawl snapshots, minimal filtering, lots of quality signals", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 47}, {"name": "llama", "filename": "lecture_11.py", "lineno": 350}], "https://github.com/togethercomputer/RedPajama-Data", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 48}, {"name": "refinedweb", "filename": "lecture_11.py", "lineno": 354}], "RefinedWeb [Falcon LLM team, 2023]", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 48}, {"name": "refinedweb", "filename": "lecture_11.py", "lineno": 354}], "https://arxiv.org/pdf/2306.01116", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 48}, {"name": "refinedweb", "filename": "lecture_11.py", "lineno": 355}], "Point: web data is all you need", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 48}, {"name": "refinedweb", "filename": "lecture_11.py", "lineno": 356}], "- trafilatura for HTML->text, extract content (WARC instead of WET files)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 48}, {"name": "refinedweb", "filename": "lecture_11.py", "lineno": 357}], "- Filtering: Gopher rules, avoid ML-based filtering to avoid biases", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 48}, {"name": "refinedweb", "filename": "lecture_11.py", "lineno": 358}], "- Fuzzy deduplication using MinHash over 5-grams", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 48}, {"name": "refinedweb", "filename": "lecture_11.py", "lineno": 359}], "Release 600B (out of 5T) tokens", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 48}, {"name": "refinedweb", "filename": "lecture_11.py", "lineno": 361}], "FineWeb [2024]", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 48}, {"name": "refinedweb", "filename": "lecture_11.py", "lineno": 361}], "https://huggingface.co/datasets/HuggingFaceFW/fineweb", {"color": "gray"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 48}, {"name": "refinedweb", "filename": "lecture_11.py", "lineno": 362}], "- Started as a replication of RefinedWeb, but improved it", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 48}, {"name": "refinedweb", "filename": "lecture_11.py", "lineno": 363}], "- 95 Common Crawl dumps", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 48}, {"name": "refinedweb", "filename": "lecture_11.py", "lineno": 364}], "- URL filtering, language ID (keep if p(en) > 0.65)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 48}, {"name": "refinedweb", "filename": "lecture_11.py", "lineno": 365}], "- Filtering: Gopher, C4, more manual rules", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 48}, {"name": "refinedweb", "filename": "lecture_11.py", "lineno": 366}], "- Fuzzy deduplication via MinHash", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 48}, {"name": "refinedweb", "filename": "lecture_11.py", "lineno": 367}], "- Anonymize email and public IP addresses (PII)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 48}, {"name": "refinedweb", "filename": "lecture_11.py", "lineno": 368}], "Result: 15T tokens", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 49}, {"name": "dolma", "filename": "lecture_11.py", "lineno": 372}], "Dolma [Soldaini+ 2024 (AI2)]", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 49}, {"name": "dolma", "filename": "lecture_11.py", "lineno": 372}], "https://arxiv.org/pdf/2402.00159", {"color": "gray"})
addImage([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 49}, {"name": "dolma", "filename": "lecture_11.py", "lineno": 373}], "https://miro.medium.com/v2/resize:fit:1400/1*-0Qqhvu7JD6Y9JgsfKJdxw.png", {"width": "100.0%"})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 49}, {"name": "dolma", "filename": "lecture_11.py", "lineno": 375}], "Reddit: from the Pushshift project (2005-2023), include submissions and comments separately", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 49}, {"name": "dolma", "filename": "lecture_11.py", "lineno": 376}], "PeS2o: 40M academic papers from Semantic Scholar", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 49}, {"name": "dolma", "filename": "lecture_11.py", "lineno": 377}], "C4, Project Gutenberg, Wikipedia/Wikibooks", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 49}, {"name": "dolma", "filename": "lecture_11.py", "lineno": 379}], "Common Crawl processing", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 49}, {"name": "dolma", "filename": "lecture_11.py", "lineno": 380}], "- Language identification (fastText classifier), keep English", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 49}, {"name": "dolma", "filename": "lecture_11.py", "lineno": 381}], "- Quality filtering (Gopher, C4 rules), avoid model-based filtering", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 49}, {"name": "dolma", "filename": "lecture_11.py", "lineno": 382}], "- Toxicity filtering using rules and Jigsaw classifier", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 49}, {"name": "dolma", "filename": "lecture_11.py", "lineno": 383}], "- Deduplication using Bloom filters", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 49}, {"name": "dolma", "filename": "lecture_11.py", "lineno": 385}], "Result: 3T tokens", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 51}], "## Summary", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 53}], "Key lesson: Data does not fall from the sky. You have to work to get it.", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 55}], "Live services (Web, Reddit, StackExchange, GitHub)", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 56}], "Lots of ad-hoc processing / filtering", {})
addText([{"name": "lecture_11", "filename": "lecture_11.py", "lineno": 58}], "Questions", {})