{"entities":[{"id":"scaling-laws","name":"Scaling Laws for Neural Language Models","type":"document"},{"id":"qwen-report","name":"Qwen Technical Report","type":"document"},{"id":"ernie-3-report","name":"ERNIE 3.0: Large-scale Knowledge Enhanced Pre-training for Language Understanding and Generation","type":"document"},{"id":"ernie-3-titan-report","name":"ERNIE 3.0 Titan: Exploring Larger-scale Knowledge Enhanced Pre-training for Language Understanding and Generation","type":"document"},{"id":"naman-goyal","name":"Naman Goyal","type":"person"},{"id":"kp-evaluating-large-language-models-trained-2021","name":"Evaluating Large Language Models Trained on Code","type":"document"},{"id":"kp-roberta-robustly-optimized-bert-pretraining-2019","name":"RoBERTa: A Robustly Optimized BERT Pretraining Approach","type":"document","subtype":"Research paper","description":"Work-level record. Its arXiv versions and any conference or journal version are listed as sources, each with its own date. A citation to the work does not identify which version the citing author read.","sourceIds":["source-kp-roberta-robustly-optimized-bert-pretraining-2019-v1"]},{"id":"luke-zettlemoyer","name":"Luke Zettlemoyer","type":"person"}],"claims":[{"id":"kp-states-roberta-robustly-optimized-bert-pretraining-2019-1","subject":"kp-roberta-robustly-optimized-bert-pretraining-2019","predicate":"states_contribution","object":{"kind":"literal","value":"The paper presents a replication study of BERT pretraining that measures the impact of hyperparameters and training data size."},"topic":"ideas","statement":"The paper presents a replication study of BERT pretraining that measures the impact of hyperparameters and training data size.","time":{"kind":"event","date":"2019-07-26","basis":"publication","note":"Date of the arXiv version quoted, submitted 26 July 2019 (UTC); later versions may word it differently."},"derivation":"explicit","resolution":"supported","editorialState":"seed","evidenceIds":["evidence-kp-states-roberta-robustly-optimized-bert-pretraining-2019-1-1"]},{"id":"kp-states-roberta-robustly-optimized-bert-pretraining-2019-2","subject":"kp-roberta-robustly-optimized-bert-pretraining-2019","predicate":"states_contribution","object":{"kind":"literal","value":"The authors state that BERT was significantly undertrained and propose a training recipe, RoBERTa, which they report can match or exceed post-BERT methods."},"topic":"ideas","statement":"The authors state that BERT was significantly undertrained and propose a training recipe, RoBERTa, which they report can match or exceed post-BERT methods.","time":{"kind":"event","date":"2019-07-26","basis":"publication","note":"Date of the arXiv version quoted, submitted 26 July 2019 (UTC); later versions may word it differently."},"derivation":"explicit","resolution":"supported","editorialState":"seed","evidenceIds":["evidence-kp-states-roberta-robustly-optimized-bert-pretraining-2019-2-1"]},{"id":"kp-cites-scaling-laws-roberta-robustly-optimized-bert-pretraining-2019","subject":"scaling-laws","predicate":"cites","object":{"kind":"entity","entityId":"kp-roberta-robustly-optimized-bert-pretraining-2019"},"topic":"ideas","statement":"Scaling Laws for Neural Language Models cites RoBERTa: A Robustly Optimized BERT Pretraining Approach.","time":{"kind":"event","date":"2020-01-23","basis":"publication","note":"Date of the citing source as recorded in the dataset; unknown where the dataset has none."},"derivation":"explicit","resolution":"supported","editorialState":"seed","evidenceIds":["evidence-kp-cites-scaling-laws-roberta-robustly-optimized-bert-pretraining-2019-1"]},{"id":"kp-cites-qwen-report-roberta-robustly-optimized-bert-pretraining-2019","subject":"qwen-report","predicate":"cites","object":{"kind":"entity","entityId":"kp-roberta-robustly-optimized-bert-pretraining-2019"},"topic":"ideas","statement":"Qwen Technical Report cites RoBERTa: A Robustly Optimized BERT Pretraining Approach.","time":{"kind":"event","date":"2023-09-28","basis":"publication","note":"Date of the citing source as recorded in the dataset; unknown where the dataset has none."},"derivation":"explicit","resolution":"supported","editorialState":"seed","evidenceIds":["evidence-kp-cites-qwen-report-roberta-robustly-optimized-bert-pretraining-2019-1"]},{"id":"kp-cites-ernie-3-report-roberta-robustly-optimized-bert-pretraining-2019","subject":"ernie-3-report","predicate":"cites","object":{"kind":"entity","entityId":"kp-roberta-robustly-optimized-bert-pretraining-2019"},"topic":"ideas","statement":"ERNIE 3.0: Large-scale Knowledge Enhanced Pre-training for Language Understanding and Generation cites RoBERTa: A Robustly Optimized BERT Pretraining Approach.","time":{"kind":"event","date":"2021-07-05","basis":"publication","note":"Date of the citing source as recorded in the dataset; unknown where the dataset has none."},"derivation":"explicit","resolution":"supported","editorialState":"seed","evidenceIds":["evidence-kp-cites-ernie-3-report-roberta-robustly-optimized-bert-pretraining-2019-1"]},{"id":"kp-cites-ernie-3-titan-report-roberta-robustly-optimized-bert-pretraining-2019","subject":"ernie-3-titan-report","predicate":"cites","object":{"kind":"entity","entityId":"kp-roberta-robustly-optimized-bert-pretraining-2019"},"topic":"ideas","statement":"ERNIE 3.0 Titan: Exploring Larger-scale Knowledge Enhanced Pre-training for Language Understanding and Generation cites RoBERTa: A Robustly Optimized BERT Pretraining Approach.","time":{"kind":"event","date":"2021-12-23","basis":"publication","note":"Date of the citing source as recorded in the dataset; unknown where the dataset has none."},"derivation":"explicit","resolution":"supported","editorialState":"seed","evidenceIds":["evidence-kp-cites-ernie-3-titan-report-roberta-robustly-optimized-bert-pretraining-2019-1"]},{"id":"kp-cites-lg-src-goo-codex-v1-roberta-robustly-optimized-bert-pretraining-2019","subject":"kp-evaluating-large-language-models-trained-2021","predicate":"cites","object":{"kind":"entity","entityId":"kp-roberta-robustly-optimized-bert-pretraining-2019"},"topic":"ideas","statement":"Evaluating Large Language Models Trained on Code (arXiv:2107.03374v1) cites RoBERTa: A Robustly Optimized BERT Pretraining Approach.","time":{"kind":"event","date":"2021-07-07","basis":"publication","note":"Date of the citing source as recorded in the dataset; unknown where the dataset has none."},"derivation":"explicit","resolution":"supported","editorialState":"seed","evidenceIds":["evidence-kp-cites-lg-src-goo-codex-v1-roberta-robustly-optimized-bert-pretraining-2019-1"]},{"id":"kp-authored-naman-goyal-roberta-robustly-optimized-bert-pretraining-2019","subject":"naman-goyal","predicate":"authored","object":{"kind":"entity","entityId":"kp-roberta-robustly-optimized-bert-pretraining-2019"},"topic":"ideas","statement":"Naman Goyal is a credited author of RoBERTa: A Robustly Optimized BERT Pretraining Approach.","time":{"kind":"event","date":"2019-07-26","basis":"publication","note":"Date of the work’s earliest recorded version, arXiv version 1, submitted 26 July 2019 (UTC)."},"derivation":"explicit","resolution":"supported","editorialState":"seed","evidenceIds":["evidence-kp-authored-naman-goyal-roberta-robustly-optimized-bert-pretraining-2019-1"]},{"id":"kp-authored-luke-zettlemoyer-roberta-robustly-optimized-bert-pretraining-2019","subject":"luke-zettlemoyer","predicate":"authored","object":{"kind":"entity","entityId":"kp-roberta-robustly-optimized-bert-pretraining-2019"},"topic":"ideas","statement":"Luke Zettlemoyer is a credited author of RoBERTa: A Robustly Optimized BERT Pretraining Approach.","time":{"kind":"event","date":"2019-07-26","basis":"publication","note":"Date of the work’s earliest recorded version, arXiv version 1, submitted 26 July 2019 (UTC)."},"derivation":"explicit","resolution":"supported","editorialState":"seed","evidenceIds":["evidence-kp-authored-luke-zettlemoyer-roberta-robustly-optimized-bert-pretraining-2019-1"]}],"sources":[{"id":"source-kp-roberta-robustly-optimized-bert-pretraining-2019-abs","documentId":"kp-roberta-robustly-optimized-bert-pretraining-2019","title":"RoBERTa: A Robustly Optimized BERT Pretraining Approach","authors":["arXiv"],"publisher":"arXiv","publicationDate":"2019-07-26","version":"arXiv abstract page, retrieved for the candidate pack on 2026-10-28 (SHA-256 a3dfb0c2a8a3a7b2edd1237857ce0fb27b9211ff93d07e61e5951ddfaec7c270). Submission history as arXiv records it (UTC): v1 26 July 2019. The page serves the latest version; each version is a separate expression, and only the one a record names was read.","evidenceUrl":"https://arxiv.org/abs/1907.11692","language":"en"},{"id":"source-kp-roberta-robustly-optimized-bert-pretraining-2019-v1","documentId":"kp-roberta-robustly-optimized-bert-pretraining-2019","title":"RoBERTa: A Robustly Optimized BERT Pretraining Approach","authors":["Yinhan Liu","Myle Ott","Naman Goyal","Jingfei Du","Mandar Joshi","Danqi Chen","Omer Levy","Mike Lewis","Luke Zettlemoyer","Veselin Stoyanov"],"publisher":"arXiv","publicationDate":"2019-07-26","version":"arXiv:1907.11692v1, submitted 26 July 2019 (UTC). Retrieved for the candidate pack on 2026-10-28 (SHA-256 76a3872d244793563a5b000b818cbaf0ca8972ab1145d32be474c05b2a8f3070). The author list is the byline as printed on the first page.","evidenceUrl":"https://arxiv.org/pdf/1907.11692v1","language":"en"}],"evidence":[{"id":"evidence-kp-states-roberta-robustly-optimized-bert-pretraining-2019-1-1","claimId":"kp-states-roberta-robustly-optimized-bert-pretraining-2019-1","sourceId":"source-kp-roberta-robustly-optimized-bert-pretraining-2019-v1","sourceRole":"primary","support":"supports","locator":"arXiv v1, PDF p. 1, abstract","passage":"We present a replication study of BERT pretraining (Devlin et al., 2019) that carefully measures the impact of many key hyperparameters and training data size.","verification":"machine_checked"},{"id":"evidence-kp-states-roberta-robustly-optimized-bert-pretraining-2019-2-1","claimId":"kp-states-roberta-robustly-optimized-bert-pretraining-2019-2","sourceId":"source-kp-roberta-robustly-optimized-bert-pretraining-2019-v1","sourceRole":"primary","support":"supports","locator":"arXiv v1, PDF p. 1, §1 Introduction, para. 2","passage":"We find that BERT was significantly undertrained and propose an improved recipe for training BERT models, which we call RoBERTa, that can match or exceed the performance of all of the post-BERT methods.","verification":"machine_checked"},{"id":"evidence-kp-cites-scaling-laws-roberta-robustly-optimized-bert-pretraining-2019-1","claimId":"kp-cites-scaling-laws-roberta-robustly-optimized-bert-pretraining-2019","sourceId":"source-scaling-laws-v1-2020","sourceRole":"primary","support":"supports","verification":"machine_checked"},{"id":"evidence-kp-cites-qwen-report-roberta-robustly-optimized-bert-pretraining-2019-1","claimId":"kp-cites-qwen-report-roberta-robustly-optimized-bert-pretraining-2019","sourceId":"source-cn-qwen","sourceRole":"primary","support":"supports","verification":"machine_checked"},{"id":"evidence-kp-cites-ernie-3-report-roberta-robustly-optimized-bert-pretraining-2019-1","claimId":"kp-cites-ernie-3-report-roberta-robustly-optimized-bert-pretraining-2019","sourceId":"c2-src-ernie3-arxiv-v1","sourceRole":"primary","support":"supports","verification":"machine_checked"},{"id":"evidence-kp-cites-ernie-3-titan-report-roberta-robustly-optimized-bert-pretraining-2019-1","claimId":"kp-cites-ernie-3-titan-report-roberta-robustly-optimized-bert-pretraining-2019","sourceId":"c2-src-ernie3-titan-arxiv-v1","sourceRole":"primary","support":"supports","verification":"machine_checked"},{"id":"evidence-kp-cites-lg-src-goo-codex-v1-roberta-robustly-optimized-bert-pretraining-2019-1","claimId":"kp-cites-lg-src-goo-codex-v1-roberta-robustly-optimized-bert-pretraining-2019","sourceId":"lg-src-goo-codex-v1","sourceRole":"primary","support":"supports","verification":"machine_checked"},{"id":"evidence-kp-authored-naman-goyal-roberta-robustly-optimized-bert-pretraining-2019-1","claimId":"kp-authored-naman-goyal-roberta-robustly-optimized-bert-pretraining-2019","sourceId":"source-kp-roberta-robustly-optimized-bert-pretraining-2019-v1","sourceRole":"primary","support":"supports","verification":"machine_checked"},{"id":"evidence-kp-authored-luke-zettlemoyer-roberta-robustly-optimized-bert-pretraining-2019-1","claimId":"kp-authored-luke-zettlemoyer-roberta-robustly-optimized-bert-pretraining-2019","sourceId":"source-kp-roberta-robustly-optimized-bert-pretraining-2019-v1","sourceRole":"primary","support":"supports","verification":"machine_checked"}],"at":{"entities":[85,183,947,948,1293,1447,1500,1553],"claims":[3832,3833,4391,4392,4393,4394,4395,4659,4660],"sources":[1377,1378],"evidence":[5290,5291,5849,5850,5851,5852,5853,6117,6118]},"index":{"byClaim":[],"marks":[],"chronicle":{"years":[{"year":1863,"topics":{"ideas":5},"stated":0},{"year":1865,"topics":{"ideas":1},"stated":0},{"year":1872,"topics":{"ideas":7},"stated":0},{"year":1943,"topics":{"ideas":1},"stated":0},{"year":1947,"topics":{"ideas":2},"stated":0},{"year":1950,"topics":{"ideas":7},"stated":0},{"year":1951,"topics":{"ideas":5},"stated":0},{"year":1952,"topics":{"ideas":2},"stated":0},{"year":1955,"topics":{"ideas":31},"stated":0},{"year":1956,"topics":{"ideas":1},"stated":0},{"year":1957,"topics":{"ideas":1},"stated":0},{"year":1958,"topics":{"ideas":3},"stated":0},{"year":1959,"topics":{"ideas":6},"stated":0},{"year":1960,"topics":{"ideas":11},"stated":0},{"year":1961,"topics":{"ideas":10},"stated":0},{"year":1965,"topics":{"ideas":8},"stated":0},{"year":1966,"topics":{"ideas":20},"stated":0},{"year":1968,"topics":{"ideas":5},"stated":0},{"year":1969,"topics":{"ideas":2},"stated":0},{"year":1971,"topics":{"ideas":4},"stated":0},{"year":1972,"topics":{"ideas":1},"stated":0},{"year":1973,"topics":{"ideas":2},"stated":0},{"year":1974,"topics":{"ideas":2},"stated":0},{"year":1975,"topics":{"ideas":1},"stated":0},{"year":1976,"topics":{"ideas":1},"stated":0},{"year":1977,"topics":{"people":2,"ideas":8},"stated":0},{"year":1979,"topics":{"ideas":4},"stated":0},{"year":1980,"topics":{"ideas":2},"stated":0},{"year":1981,"topics":{"people":2},"stated":0},{"year":1982,"topics":{"ideas":4},"stated":0},{"year":1983,"topics":{"ideas":4},"stated":0},{"year":1984,"topics":{"ideas":21},"stated":0},{"year":1985,"topics":{"ideas":9},"stated":0},{"year":1986,"topics":{"ideas":18},"stated":0},{"year":1987,"topics":{"ideas":1},"stated":0},{"year":1988,"topics":{"ideas":6},"stated":0},{"year":1989,"topics":{"ideas":8},"stated":0},{"year":1990,"topics":{"ideas":5},"stated":0},{"year":1991,"topics":{"ideas":11},"stated":0},{"year":1992,"topics":{"ideas":9},"stated":0},{"year":1993,"topics":{"ideas":14,"people":3},"stated":0},{"year":1994,"topics":{"ideas":14},"stated":0},{"year":1995,"topics":{"ideas":10,"people":1},"stated":0},{"year":1996,"topics":{"ideas":71},"stated":0},{"year":1997,"topics":{"ideas":12},"stated":0},{"year":1998,"topics":{"ideas":49},"stated":0},{"year":1999,"topics":{"ideas":27,"people":1,"institutions":1},"stated":0},{"year":2000,"topics":{"ideas":62},"stated":0},{"year":2001,"topics":{"ideas":39},"stated":0},{"year":2002,"topics":{"ideas":30},"stated":0},{"year":2003,"topics":{"ideas":8},"stated":0},{"year":2004,"topics":{"ideas":12,"people":3},"stated":0},{"year":2005,"topics":{"ideas":10,"people":1},"stated":0},{"year":2006,"topics":{"ideas":26,"people":1},"stated":0},{"year":2007,"topics":{"ideas":20},"stated":0},{"year":2008,"topics":{"ideas":57,"people":3,"institutions":1},"stated":0},{"year":2009,"topics":{"ideas":10,"people":1},"stated":0},{"year":2010,"topics":{"people":3,"ideas":11,"institutions":1,"governance":4},"stated":0},{"year":2011,"topics":{"people":3,"ideas":7,"institutions":2},"stated":0},{"year":2012,"topics":{"people":4,"ideas":17,"institutions":2},"stated":0},{"year":2013,"topics":{"ideas":46,"people":4,"institutions":2,"capital":1},"stated":0},{"year":2014,"topics":{"institutions":2,"ideas":54,"people":5},"stated":0},{"year":2015,"topics":{"people":22,"ideas":122,"institutions":3,"capital":1},"stated":0},{"year":2016,"topics":{"ideas":88,"people":2},"stated":0},{"year":2017,"topics":{"people":16,"institutions":2,"ideas":128,"governance":2},"stated":0},{"year":2018,"topics":{"people":8,"ideas":158,"institutions":8,"governance":1},"stated":0},{"year":2019,"topics":{"capital":1,"compute":1,"people":10,"governance":19,"ideas":175,"institutions":4},"stated":0},{"year":2020,"topics":{"people":6,"ideas":98,"institutions":1,"governance":2},"stated":0},{"year":2021,"topics":{"people":27,"ideas":177,"compute":1,"institutions":23,"governance":4,"capital":6},"stated":0},{"year":2022,"topics":{"people":15,"ideas":217,"institutions":6,"governance":3,"compute":2,"capital":1},"stated":0},{"year":2023,"topics":{"capital":5,"compute":7,"institutions":23,"people":51,"governance":179,"ideas":389},"stated":0},{"year":2024,"topics":{"capital":12,"compute":19,"governance":282,"people":80,"ideas":178,"institutions":28},"stated":1},{"year":2025,"topics":{"governance":547,"compute":69,"capital":34,"people":90,"institutions":18,"ideas":178},"stated":7},{"year":2026,"topics":{"governance":442,"capital":8,"compute":32,"institutions":21,"ideas":91,"people":107},"stated":19},{"year":2027,"topics":{},"stated":9},{"year":2028,"topics":{},"stated":6},{"year":2029,"topics":{},"stated":3},{"year":2030,"topics":{},"stated":3},{"year":2032,"topics":{},"stated":2},{"year":2035,"topics":{},"stated":2},{"year":2039,"topics":{},"stated":1},{"year":2042,"topics":{},"stated":1},{"year":2050,"topics":{},"stated":1}],"scheduled":[],"statedBy":[]},"collections":[{"target":{"kind":"propositions","id":"superintelligence"},"title":"Propositions about superintelligence, 1863–2022","claimIds":[],"records":[]},{"target":{"kind":"propositions","id":"risk-debate"},"title":"Arguments about catastrophic risk from advanced AI","claimIds":[],"records":[]},{"target":{"kind":"propositions","id":"yudkowsky-arguments"},"title":"Eliezer Yudkowsky’s arguments on common claims about AI, 1996–2026","claimIds":[],"records":[]},{"target":{"kind":"propositions","id":"economics-of-machine-intelligence"},"title":"Economics of machine intelligence","claimIds":[],"records":[]},{"target":{"kind":"propositions","id":"orbital-compute-arguments"},"title":"Arguments about compute in orbit","claimIds":[],"records":[]},{"target":{"kind":"propositions","id":"ai-bio-arguments"},"title":"Arguments about AI and biological risk","claimIds":[],"records":[]},{"target":{"kind":"propositions","id":"ai-in-science"},"title":"AI in science","claimIds":[],"records":[]},{"target":{"kind":"documents"},"title":"Frontier safety frameworks","claimIds":[]},{"target":{"kind":"compare","id":"frontier-safety-frameworks"},"title":"Compare frameworks","claimIds":[]},{"target":{"kind":"control","id":"openai"},"title":"OpenAI’s control structure, 2015–2026","claimIds":[]},{"target":{"kind":"control","id":"anthropic"},"title":"Anthropic’s control structure, 2021–2026","claimIds":[]},{"target":{"kind":"commitments"},"title":"External commitments and obligations","claimIds":[]},{"target":{"kind":"question","id":"frameworks-to-obligations"},"title":"Frameworks, commitments and legal obligations, 2023–2026","claimIds":[]},{"target":{"kind":"reading"},"title":"Founding and early leadership","claimIds":[]}]},"version":"131evrz"}