{"entry_id": "a64701c3-9c8c-4661-a34d-e4d86dd748f8", "infos": [{"txt": "header", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 0, "bbox": [[0.042, 0.007, 0.062, 0.021]]}]}, "tags": ["header"], "label": "O", "web_segment_id": 1, "global_sentence_id": 0, "edu_l1_label": "EDU_O"}, {"txt": "header", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 0, "bbox": [[0.064, 0.011, 0.149, 0.018]]}]}, "tags": ["header"], "label": "O", "web_segment_id": 2, "global_sentence_id": 1, "edu_l1_label": "EDU_O"}, {"txt": "header", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 0, "bbox": [[0.737, 0.056, 0.837, 0.069]]}]}, "tags": ["header"], "label": "O", "web_segment_id": 3, "global_sentence_id": 2, "edu_l1_label": "EDU_O"}, {"txt": "header", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 0, "bbox": [[0.102, 0.074, 0.827, 0.088]]}]}, "tags": ["header"], "label": "O", "web_segment_id": 4, "global_sentence_id": 3, "edu_l1_label": "EDU_O"}, {"txt": "Clinical Trials", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 100000, "bbox": [[0.736, 0.108, 0.793, 0.115]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 5, "global_sentence_id": 4, "edu_l1_label": "EDU_O"}, {"txt": "1-6", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 200000, "bbox": [[0.736, 0.12, 0.754, 0.126]]}]}, "tags": ["text"], "label": "O", "web_segment_id": 6, "global_sentence_id": 5, "edu_l1_label": "EDU_O"}, {"txt": "©The Author(s) 2025", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 200000, "bbox": [[0.736, 0.13, 0.843, 0.137]]}]}, "tags": ["text"], "label": "O", "web_segment_id": 7, "global_sentence_id": 6, "edu_l1_label": "EDU_O"}, {"txt": "figure", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 200000, "bbox": [[0.736, 0.137, 0.793, 0.151]]}]}, "tags": ["figure"], "label": "O", "web_segment_id": 8, "global_sentence_id": 7, "edu_l1_label": "EDU_O"}, {"txt": "Article reuse guidelines:", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 200000, "bbox": [[0.736, 0.155, 0.85, 0.16]]}]}, "tags": ["text"], "label": "O", "web_segment_id": 9, "global_sentence_id": 8, "edu_l1_label": "EDU_O"}, {"txt": "From RAGs to riches:", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 300000, "bbox": [[0.104, 0.13, 0.407, 0.146]]}]}, "tags": ["title"], "label": "article_title", "web_segment_id": 10, "global_sentence_id": 9, "edu_l1_label": "BOT"}, {"txt": " Utilizing large language models to write documents for clinical trials", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 300000, "bbox": [[0.407, 0.13, 0.61, 0.147], [0.103, 0.156, 0.619, 0.172], [0.102, 0.181, 0.326, 0.198]]}]}, "tags": ["title"], "label": "article_title", "web_segment_id": 10, "global_sentence_id": 10, "edu_l1_label": "BOT"}, {"txt": "sagepub.com/journals-permissions", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 400000, "bbox": [[0.736, 0.166, 0.895, 0.171]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 11, "global_sentence_id": 11, "edu_l1_label": "EDU_O"}, {"txt": "DOI:", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 500000, "bbox": [[0.736, 0.174, 0.761, 0.181]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 12, "global_sentence_id": 12, "edu_l1_label": "EDU_O"}, {"txt": " 10.1177/17407745251320806", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 500000, "bbox": [[0.761, 0.174, 0.903, 0.181]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 12, "global_sentence_id": 13, "edu_l1_label": "EDU_O"}, {"txt": "journals.sagepub.com/home/ctj", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 600000, "bbox": [[0.738, 0.187, 0.882, 0.193]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 13, "global_sentence_id": 14, "edu_l1_label": "EDU_O"}, {"txt": "S Sage", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 600000, "bbox": [[0.736, 0.196, 0.799, 0.208]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 14, "global_sentence_id": 15, "edu_l1_label": "EDU_O"}, {"txt": "figure", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 700000, "bbox": [[0.733, 0.193, 0.754, 0.213]]}]}, "tags": ["figure"], "label": "figure", "web_segment_id": 15, "global_sentence_id": 16, "edu_l1_label": "EDU_O"}, {"txt": "Nigel Markey', Ilyass El-Mansouri2,Gaetan Rensonnet2,Casper van Langen' and Christoph Meier", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 800000, "bbox": [[0.105, 0.249, 0.62, 0.262], [0.104, 0.268, 0.494, 0.273]]}]}, "tags": ["text"], "label": "author", "web_segment_id": 16, "global_sentence_id": 17, "edu_l1_label": "EDU_O"}, {"txt": "Abstract", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 900000, "bbox": [[0.103, 0.329, 0.172, 0.338]]}]}, "tags": ["title"], "label": "abstract", "web_segment_id": 17, "global_sentence_id": 18, "edu_l1_label": "BOS"}, {"txt": "Background/Aims:", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 900000, "bbox": [[0.104, 0.344, 0.249, 0.353]]}]}, "tags": ["text"], "label": "abstract", "web_segment_id": 18, "global_sentence_id": 19, "edu_l1_label": "IOS"}, {"txt": " Clinical trials require numerous documents to be written: Protocols, consent forms, clinical study reports, and many others.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 900000, "bbox": [[0.249, 0.344, 0.914, 0.354], [0.104, 0.361, 0.288, 0.368]]}]}, "tags": ["text"], "label": "abstract", "web_segment_id": 18, "global_sentence_id": 20, "edu_l1_label": "IOS"}, {"txt": "Large language models offer the potential to rapidly generate first-draft versions of these documents; however, there are concerns about the quality of their output.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 900000, "bbox": [[0.288, 0.361, 0.915, 0.367], [0.103, 0.375, 0.615, 0.383]]}]}, "tags": ["text"], "label": "abstract", "web_segment_id": 18, "global_sentence_id": 21, "edu_l1_label": "IOS"}, {"txt": "Here, we report an evaluation of how good large language models are at generating sections of one such document, clinical trial protocols.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 900000, "bbox": [[0.615, 0.375, 0.915, 0.384], [0.104, 0.39, 0.74, 0.398]]}]}, "tags": ["text"], "label": "abstract", "web_segment_id": 18, "global_sentence_id": 22, "edu_l1_label": "IOS"}, {"txt": "Methods:", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 900000, "bbox": [[0.105, 0.404, 0.178, 0.413]]}]}, "tags": ["text"], "label": "abstract", "web_segment_id": 19, "global_sentence_id": 23, "edu_l1_label": "IOS"}, {"txt": " Using an off-the-shelf large language model, we generated protocol sections for a broad range of diseases and clinical trial phases.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 900000, "bbox": [[0.178, 0.404, 0.915, 0.414], [0.104, 0.421, 0.242, 0.428]]}]}, "tags": ["text"], "label": "abstract", "web_segment_id": 19, "global_sentence_id": 24, "edu_l1_label": "IOS"}, {"txt": "Each of these document sections we assessed across four dimensions: Clinical thinking and logic;Transparency and references; Medical and clinical terminology; and Content relevance and suitability.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 900000, "bbox": [[0.242, 0.421, 0.917, 0.43], [0.105, 0.435, 0.741, 0.443]]}]}, "tags": ["text"], "label": "abstract", "web_segment_id": 19, "global_sentence_id": 25, "edu_l1_label": "IOS"}, {"txt": "To improve performance,we used the retrieval-augmented generation method to enhance the large language model with accurate up-to-date information, including regulatory guidance documents and data from ClinicalTrials.gov.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 900000, "bbox": [[0.741, 0.435, 0.915, 0.445], [0.104, 0.452, 0.915, 0.458], [0.104, 0.466, 0.703, 0.474]]}]}, "tags": ["text"], "label": "abstract", "web_segment_id": 19, "global_sentence_id": 26, "edu_l1_label": "IOS"}, {"txt": "Using this retrieval-augmented generationlarge language model, we regenerated the same protocol sections and assessed them across the same four dimensions.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 900000, "bbox": [[0.703, 0.466, 0.915, 0.474], [0.104, 0.482, 0.916, 0.489], [0.104, 0.497, 0.184, 0.504]]}]}, "tags": ["text"], "label": "abstract", "web_segment_id": 19, "global_sentence_id": 27, "edu_l1_label": "IOS"}, {"txt": "Results:", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 900000, "bbox": [[0.102, 0.51, 0.919, 0.612]]}]}, "tags": ["text"], "label": "abstract", "web_segment_id": 20, "global_sentence_id": 28, "edu_l1_label": "IOS"}, {"txt": " We find that the off-the-shelf large language model delivers reasonable results, especially when assessing content relevance and the correct use of medical and clinical terminology, with scores of over $80\\%$.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 900000, "bbox": [[0.102, 0.51, 0.919, 0.612]]}]}, "tags": ["text"], "label": "abstract", "web_segment_id": 20, "global_sentence_id": 29, "edu_l1_label": "IOS"}, {"txt": "However, the off-the-shelf large language model shows limited performance in clinical thinking and logic and transparency and references, with assessment scores of$\\approx 40\\%$or less.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 900000, "bbox": [[0.102, 0.51, 0.919, 0.612]]}]}, "tags": ["text"], "label": "abstract", "web_segment_id": 20, "global_sentence_id": 30, "edu_l1_label": "IOS"}, {"txt": "The use of retrieval-augmented generation substantially improves the writing quality of the large language model, with clinical thinking and logic and transparency and references scores increasing to$\\approx 80\\%$.The retrieval-augmented generation method thus greatly improves the practical usability of large language models for clinical trial-related writing.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 900000, "bbox": [[0.102, 0.51, 0.919, 0.612]]}]}, "tags": ["text"], "label": "abstract", "web_segment_id": 20, "global_sentence_id": 31, "edu_l1_label": "IOS"}, {"txt": "Discussion:", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 900000, "bbox": [[0.104, 0.616, 0.191, 0.625]]}]}, "tags": ["text"], "label": "abstract", "web_segment_id": 21, "global_sentence_id": 32, "edu_l1_label": "IOS"}, {"txt": " Our results suggest that hybrid large language model architectures, such as the retrieval-augmented generation method we utilized, offer strong potential for clinical trial-related writing, including a wide variety of docu-ments.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 900000, "bbox": [[0.191, 0.616, 0.915, 0.625], [0.103, 0.634, 0.916, 0.64], [0.113, 0.649, 0.15, 0.655]]}]}, "tags": ["text"], "label": "abstract", "web_segment_id": 21, "global_sentence_id": 33, "edu_l1_label": "IOS"}, {"txt": "This is potentially transformative, since it addresses several major bottlenecks of drug development.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 900000, "bbox": [[0.15, 0.649, 0.827, 0.655]]}]}, "tags": ["text"], "label": "abstract", "web_segment_id": 21, "global_sentence_id": 34, "edu_l1_label": "IOS"}, {"txt": "Keywords", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 1000000, "bbox": [[0.104, 0.684, 0.18, 0.693]]}]}, "tags": ["title"], "label": "abstract", "web_segment_id": 22, "global_sentence_id": 35, "edu_l1_label": "BOS"}, {"txt": "Generative artificial intelligence, large language models, clinical trial writing, clinical trial protocol, retrieval-augmented generation", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 1000000, "bbox": [[0.104, 0.698, 0.893, 0.708], [0.103, 0.716, 0.175, 0.723]]}]}, "tags": ["text"], "label": "abstract", "web_segment_id": 23, "global_sentence_id": 36, "edu_l1_label": "IOS"}, {"txt": "Background and aims", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 1100000, "bbox": [[0.103, 0.77, 0.304, 0.781]]}]}, "tags": ["title"], "label": "title1", "web_segment_id": 24, "global_sentence_id": 37, "edu_l1_label": "BOS"}, {"txt": "During clinical trials, large volumes of documents need to be written, including protocols, amendments, patient informed consent forms, clinical study reports, and many others.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 1200000, "bbox": [[0.105, 0.795, 0.499, 0.804], [0.103, 0.812, 0.499, 0.82], [0.103, 0.826, 0.497, 0.835], [0.104, 0.844, 0.202, 0.849]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 25, "global_sentence_id": 38, "edu_l1_label": "IOS"}, {"txt": "These documents are critically important for the planning and execution of trials and are often required by regulation; therefore,high-quality writing is essential.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 1200000, "bbox": [[0.202, 0.844, 0.499, 0.85], [0.104, 0.856, 0.495, 0.865], [0.104, 0.873, 0.499, 0.881], [0.104, 0.888, 0.189, 0.895]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 25, "global_sentence_id": 39, "edu_l1_label": "IOS"}, {"txt": "Specifically,clinical trial documents must be scientifically and clinically precise and accurate,with correct use of terminology, and must contain appropriate references to the literature, regulatory", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 1200000, "bbox": [[0.189, 0.888, 0.499, 0.895], [0.105, 0.901, 0.501, 0.911], [0.104, 0.919, 0.499, 0.926], [0.103, 0.935, 0.497, 0.943]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 25, "global_sentence_id": 40, "edu_l1_label": "IOS"}, {"txt": "guidelines, and other documents.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 1200000, "bbox": [[0.521, 0.772, 0.757, 0.778]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 26, "global_sentence_id": 41, "edu_l1_label": "IOS"}, {"txt": "Due to these stringent requirements, sponsors of clinical trials spend consider-able time and resources on trial-related writing.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 1200000, "bbox": [[0.757, 0.772, 0.915, 0.779], [0.521, 0.788, 0.915, 0.792], [0.521, 0.802, 0.883, 0.809]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 26, "global_sentence_id": 42, "edu_l1_label": "IOS"}, {"txt": "For example, most large pharmaceutical companies each", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 1200000, "bbox": [[0.883, 0.802, 0.916, 0.809], [0.52, 0.818, 0.915, 0.825]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 26, "global_sentence_id": 43, "edu_l1_label": "IOS"}, {"txt": "Boston Consulting Group,London, UK", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 1300000, "bbox": [[0.526, 0.858, 0.736, 0.866]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 27, "global_sentence_id": 44, "edu_l1_label": "EDU_O"}, {"txt": "2Boston Consulting Group, Paris, France", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 1400000, "bbox": [[0.52, 0.87, 0.739, 0.878]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 28, "global_sentence_id": 45, "edu_l1_label": "IOS"}, {"txt": "Corresponding author:", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 1500000, "bbox": [[0.521, 0.896, 0.663, 0.904]]}]}, "tags": ["text"], "label": "O", "web_segment_id": 29, "global_sentence_id": 46, "edu_l1_label": "EDU_O"}, {"txt": "Christoph Meier, Boston Consulting Group, 80 Charlotte Street, London WIT 4DF, UK.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 1500000, "bbox": [[0.521, 0.909, 0.911, 0.916], [0.521, 0.921, 0.603, 0.929]]}]}, "tags": ["text"], "label": "O", "web_segment_id": 30, "global_sentence_id": 47, "edu_l1_label": "EDU_O"}, {"txt": "Email:", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 1500000, "bbox": [[0.521, 0.934, 0.554, 0.941]]}]}, "tags": ["text"], "label": "O", "web_segment_id": 31, "global_sentence_id": 48, "edu_l1_label": "EDU_O"}, {"txt": " meier.chris@bcg.com", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 1500000, "bbox": [[0.554, 0.934, 0.672, 0.941]]}]}, "tags": ["text"], "label": "O", "web_segment_id": 31, "global_sentence_id": 49, "edu_l1_label": "EDU_O"}, {"txt": "header", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 1500000, "bbox": [[0.081, 0.048, 0.897, 0.064]]}]}, "tags": ["header"], "label": "O", "web_segment_id": 1, "global_sentence_id": 50, "edu_l1_label": "EDU_O"}, {"txt": "employ tens to hundreds of medical writers and reviewers.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 1600000, "bbox": [[0.08, 0.088, 0.482, 0.312]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 2, "global_sentence_id": 51, "edu_l1_label": "IOS"}, {"txt": "Even with these resources, it often takes organizations a long time to write,review,and finalize clinical trial documents.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 1600000, "bbox": [[0.08, 0.088, 0.482, 0.312]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 2, "global_sentence_id": 52, "edu_l1_label": "IOS"}, {"txt": "As an illustration, a clinical trial protocol typically has 50-150 or more pages and can take 3-6 months or longer to prepare.2 A substan-tial proportion of this time is due to the writing and reviewing process, to ensure that the document achieves the high quality expected.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 1600000, "bbox": [[0.08, 0.088, 0.482, 0.312]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 2, "global_sentence_id": 53, "edu_l1_label": "IOS"}, {"txt": "As a result, writing is one of the major rate-limiting steps in the development pro-cess.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 1600000, "bbox": [[0.08, 0.088, 0.482, 0.312]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 2, "global_sentence_id": 54, "edu_l1_label": "IOS"}, {"txt": "With pharmaceutical companies under pressure to accelerate tria$1\\mathrm {\\sim }^{3,4}$and to submit regulatory documents faster, there is strong interest across the industry in using new technologies and approaches to speed up trial-related writing.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 1600000, "bbox": [[0.08, 0.088, 0.482, 0.312]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 2, "global_sentence_id": 55, "edu_l1_label": "IOS"}, {"txt": "In the past few years, large language models (LLMs), a new class of generative artificial intelligence algorithms, have advanced to a point where they can produce near-human-quality writing.5 Since the arrival of ChatGPT,6 the first widely used tool built on LLMs,there has been interest in using these algorithms in the context of clinical trials.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 1700000, "bbox": [[0.08, 0.317, 0.482, 0.557]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 3, "global_sentence_id": 56, "edu_l1_label": "IOS"}, {"txt": "Examples include enhancing patient-trial matching,⁷ clinical trial planning,8 assisting in medical writing tasks, and others.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 1700000, "bbox": [[0.08, 0.317, 0.482, 0.557]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 3, "global_sentence_id": 57, "edu_l1_label": "IOS"}, {"txt": "While it is early days for these efforts, we see signs of great potential but also challenges, such as accuracy and potential biases in the LLMs, as well as concerns about robust-ness and reproducibility.$.1011$To help address some of these questions, we describe a framework for document quality evaluation, and we report an analysis of LLMs in the context of clinical trial-related writing.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 1700000, "bbox": [[0.08, 0.317, 0.482, 0.557]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 3, "global_sentence_id": 58, "edu_l1_label": "IOS"}, {"txt": "Methods", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 1800000, "bbox": [[0.5, 0.086, 0.579, 0.098]]}]}, "tags": ["title"], "label": "title1", "web_segment_id": 4, "global_sentence_id": 59, "edu_l1_label": "BOS"}, {"txt": "Our assessment focused on GPT-4, one of the leading LLMs available today12and utilizing it to generate key sections of clinical trial protocols.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 1900000, "bbox": [[0.5, 0.112, 0.894, 0.123], [0.5, 0.127, 0.672, 0.132], [0.676, 0.13, 0.893, 0.139], [0.499, 0.146, 0.757, 0.152]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 5, "global_sentence_id": 60, "edu_l1_label": "IOS"}, {"txt": "The LLM output was subsequently assessed in terms of writing quality.Specifically we analyzed four dimensions: Clinical thinking and logic, which measures how closely recom-mendations from regulatory guidance documents were included in the generated section; Transparency and references, which verifies the presence and relevance of cited scientific sources in the generated text; Medical and clinical terminology, which assesses the use of appropriate jargon and scales of measurements;Content relevance and suitability, which measures,among others, whether the generated protocol section was specific to the disease and trial phase.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 1900000, "bbox": [[0.757, 0.146, 0.895, 0.152], [0.5, 0.162, 0.895, 0.168], [0.499, 0.174, 0.895, 0.183], [0.5, 0.191, 0.895, 0.198], [0.5, 0.208, 0.894, 0.214], [0.499, 0.221, 0.896, 0.23], [0.499, 0.238, 0.895, 0.245], [0.499, 0.253, 0.895, 0.261], [0.499, 0.27, 0.895, 0.276], [0.499, 0.285, 0.893, 0.292], [0.5, 0.298, 0.894, 0.308], [0.499, 0.315, 0.893, 0.322], [0.5, 0.331, 0.801, 0.338]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 5, "global_sentence_id": 61, "edu_l1_label": "IOS"}, {"txt": "An overview of the methodology is given in Figure 1, and a full description of both the generation and evaluation pro-cess, including the criteria and requirements used,is provided in the Supplementary Information.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 1900000, "bbox": [[0.801, 0.331, 0.893, 0.338], [0.499, 0.346, 0.894, 0.353], [0.499, 0.36, 0.895, 0.368], [0.499, 0.378, 0.894, 0.384], [0.499, 0.393, 0.816, 0.399]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 5, "global_sentence_id": 62, "edu_l1_label": "IOS"}, {"txt": "Our analysis does not make direct comparisons between LLM-written text and fully human-written text.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 2000000, "bbox": [[0.52, 0.406, 0.895, 0.414], [0.499, 0.421, 0.893, 0.429], [0.499, 0.438, 0.534, 0.446]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 6, "global_sentence_id": 63, "edu_l1_label": "IOS"}, {"txt": "This is because, from our experience in the field,there is often substantial variability between individual human writers.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 2000000, "bbox": [[0.534, 0.438, 0.897, 0.446], [0.499, 0.453, 0.894, 0.46], [0.5, 0.467, 0.61, 0.475]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 6, "global_sentence_id": 64, "edu_l1_label": "IOS"}, {"txt": "It is therefore challenging to establish a single, objective “ground truth” to compare against.Our evaluation framework, with its four dimensions described above, addresses this challenge by breaking down the assessment into discrete sub-dimensions which can be assessed objectively.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 2000000, "bbox": [[0.61, 0.467, 0.893, 0.476], [0.499, 0.485, 0.897, 0.492], [0.499, 0.497, 0.895, 0.507], [0.499, 0.513, 0.894, 0.523], [0.499, 0.528, 0.895, 0.537], [0.499, 0.545, 0.74, 0.553]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 6, "global_sentence_id": 65, "edu_l1_label": "IOS"}, {"txt": "figure", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 2100000, "bbox": [[0.106, 0.594, 0.874, 0.699]]}]}, "tags": ["figure"], "label": "figure", "web_segment_id": 7, "global_sentence_id": 66, "edu_l1_label": "EDU_O"}, {"txt": "(c)", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 2200000, "bbox": [[0.107, 0.711, 0.128, 0.721]]}]}, "tags": ["text"], "label": "O", "web_segment_id": 8, "global_sentence_id": 67, "edu_l1_label": "EDU_O"}, {"txt": "figure", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 2300000, "bbox": [[0.106, 0.724, 0.878, 0.863]]}]}, "tags": ["figure"], "label": "figure", "web_segment_id": 9, "global_sentence_id": 68, "edu_l1_label": "EDU_O"}, {"txt": "Figure I. Overview of methodology and approach used in this analysis. (a) Typical use of off-the-shelf LLMs. (b) Retrieval-augmented generation (RAG) methodology for enhancing LLMs. (c) ClinEval methodology for assessing the output of large language models (LLMs). Further details are described in the \"Methods\" section and in the supplementary information.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 2400000, "bbox": [[0.081, 0.878, 0.888, 0.916]]}]}, "tags": ["figure_caption"], "label": "figure_title", "web_segment_id": 10, "global_sentence_id": 69, "edu_l1_label": "EDU_O"}, {"txt": "header", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 2500000, "bbox": [[0.102, 0.049, 0.918, 0.062]]}]}, "tags": ["header"], "label": "O", "web_segment_id": 1, "global_sentence_id": 70, "edu_l1_label": "EDU_O"}, {"txt": "Our assessment targeted two key sections of a clini-cal trial protocol document:", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 2600000, "bbox": [[0.124, 0.089, 0.499, 0.098], [0.103, 0.106, 0.311, 0.113]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 2, "global_sentence_id": 71, "edu_l1_label": "IOS"}, {"txt": " the endpoints section and the eligibility criteria section.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 2600000, "bbox": [[0.311, 0.106, 0.498, 0.113], [0.104, 0.12, 0.318, 0.128]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 2, "global_sentence_id": 72, "edu_l1_label": "IOS"}, {"txt": "Two LLM models were evaluated: off-the-shelf GPT-4 via its commercially available application programming interface,6,12used as a baseline; and a retrieval-augmented generation (RAG) GPT-4 as an alternative to the off-the-shelf ver-sion (see Figure 1(b)).", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 2600000, "bbox": [[0.318, 0.12, 0.5, 0.128], [0.103, 0.137, 0.498, 0.144], [0.103, 0.152, 0.458, 0.154], [0.467, 0.152, 0.498, 0.159], [0.104, 0.168, 0.498, 0.174], [0.103, 0.181, 0.499, 0.188], [0.104, 0.198, 0.259, 0.204]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 2, "global_sentence_id": 73, "edu_l1_label": "IOS"}, {"txt": "RAG is a methodology for incor-porating knowledge from external databases,13 and involves providing the LLM with external sources of knowledge to supplement the model's internal represen-tation of information.14 The RAG-augmented LLM was configured as follows: based on a user input query below, an LLM-powered decision agent automatically decided which tools to use to fetch relevant context and feed it to an LLM for final summarization and docu-ment generation.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 2600000, "bbox": [[0.259, 0.198, 0.5, 0.204], [0.104, 0.213, 0.499, 0.219], [0.104, 0.228, 0.5, 0.235], [0.103, 0.241, 0.5, 0.249], [0.104, 0.256, 0.495, 0.264], [0.104, 0.275, 0.497, 0.281], [0.105, 0.287, 0.499, 0.298], [0.103, 0.302, 0.498, 0.311], [0.104, 0.317, 0.5, 0.325], [0.104, 0.334, 0.232, 0.341]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 2, "global_sentence_id": 74, "edu_l1_label": "IOS"}, {"txt": "As described in more detail in the Supplementary Information, the following tools were utilized as part of the RAG-augmentation: vector store databases to access and analyze regulatory guidance documents; the clinicaltrials.gov AACT (Aggregate Analysis of Clinical Trials) database; SemanticsScholar connector to scrape scientific literature.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 2600000, "bbox": [[0.232, 0.334, 0.499, 0.341], [0.104, 0.347, 0.499, 0.356], [0.104, 0.365, 0.499, 0.372], [0.103, 0.378, 0.499, 0.387], [0.103, 0.393, 0.5, 0.402], [0.104, 0.408, 0.5, 0.418], [0.103, 0.426, 0.385, 0.433]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 2, "global_sentence_id": 75, "edu_l1_label": "IOS"}, {"txt": "Both the off-the-shelf GPT-4 and RAG-augmented GPT-4 were prompted with a natural-language user query of the form “Write the {section} section of a Phase {phase} clinical trial protocol in {disease}.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 2700000, "bbox": [[0.125, 0.438, 0.499, 0.448], [0.103, 0.453, 0.5, 0.463], [0.104, 0.471, 0.499, 0.478], [0.105, 0.484, 0.453, 0.493]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 3, "global_sentence_id": 76, "edu_l1_label": "IOS"}, {"txt": "Focus on FDA guidance” where section, phase,and disease were customizable.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 2700000, "bbox": [[0.453, 0.484, 0.499, 0.493], [0.103, 0.502, 0.5, 0.508], [0.105, 0.518, 0.244, 0.523]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 3, "global_sentence_id": 77, "edu_l1_label": "IOS"}, {"txt": "For each disease and trial phase,and for each model, five endpoints sections and five eligibility criteria sections were generated, with poten-tial differences between versions due to the stochastic nature of the underlying LLM models.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 2700000, "bbox": [[0.244, 0.518, 0.501, 0.525], [0.103, 0.532, 0.499, 0.538], [0.103, 0.547, 0.5, 0.554], [0.105, 0.561, 0.499, 0.569], [0.105, 0.577, 0.381, 0.584]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 3, "global_sentence_id": 78, "edu_l1_label": "IOS"}, {"txt": "The evaluation process was strictly identical for both the off-the-shelf and RAG-augmented LLMs, and", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 2800000, "bbox": [[0.125, 0.591, 0.499, 0.599], [0.103, 0.606, 0.498, 0.615]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 4, "global_sentence_id": 79, "edu_l1_label": "IOS"}, {"txt": "involved a combination of algorithmic and human expert-based scoring.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 2800000, "bbox": [[0.52, 0.089, 0.915, 0.098], [0.52, 0.106, 0.672, 0.113]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 5, "global_sentence_id": 80, "edu_l1_label": "IOS"}, {"txt": "Briefly, the algorithmic assessment consisted in prompting GPT-4, used as an evaluator LLM, with the generated protocol section or an individ-ual section element (i.e.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 2800000, "bbox": [[0.672, 0.106, 0.915, 0.113], [0.52, 0.122, 0.916, 0.128], [0.521, 0.134, 0.915, 0.144], [0.521, 0.152, 0.688, 0.158]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 5, "global_sentence_id": 81, "edu_l1_label": "IOS"}, {"txt": "an endpoint or an eligibility cri-terion) and asking it to provide a binary score for each sub-dimension based on a specific list of requirements (detailed in Supplementary Table 2).", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 2800000, "bbox": [[0.688, 0.152, 0.916, 0.157], [0.521, 0.166, 0.914, 0.174], [0.521, 0.182, 0.916, 0.188], [0.52, 0.195, 0.78, 0.203]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 5, "global_sentence_id": 82, "edu_l1_label": "IOS"}, {"txt": "For every protocol section that had been generated, metrics for each dimen-sion were then obtained as an average of the scores of the relevant sub-dimensions.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 2800000, "bbox": [[0.78, 0.195, 0.915, 0.203], [0.521, 0.213, 0.916, 0.219], [0.521, 0.227, 0.917, 0.234], [0.52, 0.241, 0.73, 0.249]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 5, "global_sentence_id": 83, "edu_l1_label": "IOS"}, {"txt": "Those requirements were developed in consultation with internal and external experts and aim to capture best human practices in clini-cal protocol writing.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 2800000, "bbox": [[0.73, 0.241, 0.916, 0.249], [0.52, 0.255, 0.915, 0.265], [0.52, 0.273, 0.915, 0.279], [0.521, 0.288, 0.665, 0.295]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 5, "global_sentence_id": 84, "edu_l1_label": "IOS"}, {"txt": "Each of the two models generated a total of 140document sections, which covered protocols for 14 dis-eases across different phases of clinical trials (see Supplementary Table 3).", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 2900000, "bbox": [[0.542, 0.3, 0.916, 0.31], [0.52, 0.316, 0.915, 0.322], [0.52, 0.334, 0.915, 0.34], [0.521, 0.346, 0.705, 0.355]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 6, "global_sentence_id": 85, "edu_l1_label": "IOS"}, {"txt": "The scores are presented as percentages which indicate the mean score achieved by the generated documents across all diseases and phases and section types.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 2900000, "bbox": [[0.705, 0.346, 0.916, 0.355], [0.521, 0.365, 0.915, 0.373], [0.521, 0.377, 0.915, 0.386], [0.52, 0.394, 0.662, 0.401]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 6, "global_sentence_id": 86, "edu_l1_label": "IOS"}, {"txt": "Because of the non-deterministic nature of LLMs, we performed five repetitions for each query combination.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 2900000, "bbox": [[0.662, 0.394, 0.916, 0.401], [0.521, 0.409, 0.914, 0.416], [0.521, 0.424, 0.662, 0.431]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 6, "global_sentence_id": 87, "edu_l1_label": "IOS"}, {"txt": "This approach mitigates the impact of inherent randomness in the model's responses.Statistical testswere performed to assess whether differ-ences in performance metrics between the two models were statistically significant.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 2900000, "bbox": [[0.662, 0.424, 0.916, 0.431], [0.521, 0.44, 0.918, 0.446], [0.521, 0.452, 0.915, 0.46], [0.52, 0.47, 0.916, 0.476], [0.521, 0.485, 0.72, 0.492]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 6, "global_sentence_id": 88, "edu_l1_label": "IOS"}, {"txt": "Results", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 3000000, "bbox": [[0.521, 0.52, 0.588, 0.531]]}]}, "tags": ["title"], "label": "title1", "web_segment_id": 7, "global_sentence_id": 89, "edu_l1_label": "BOS"}, {"txt": "An overview of the result of our assessment is shown in Figure 2.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 3100000, "bbox": [[0.518, 0.544, 0.919, 0.619]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 8, "global_sentence_id": 90, "edu_l1_label": "IOS"}, {"txt": "Overall, we find that the off-the shelf LLM delivers reasonable results, specifically good content relevance anid suitability (assessment score$>80\\%)$,and excellent medical and clinical terminology $(>99\\%),$", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 3100000, "bbox": [[0.518, 0.544, 0.919, 0.619]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 8, "global_sentence_id": 91, "edu_l1_label": "IOS"}, {"txt": "table", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 3200000, "bbox": [[0.123, 0.644, 0.897, 0.894]]}]}, "tags": ["table"], "label": "table", "web_segment_id": 9, "global_sentence_id": 92, "edu_l1_label": "EDU_O"}, {"txt": "Figure 2. Comparison of off-the-shelf LLM and RAG-augmented LLM. Further information is described in the \"Methods\" section and supplementary information.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 3300000, "bbox": [[0.102, 0.909, 0.9, 0.933]]}]}, "tags": ["figure_caption"], "label": "figure_title", "web_segment_id": 10, "global_sentence_id": 93, "edu_l1_label": "EDU_O"}, {"txt": "*Difference statistically significant", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 3400000, "bbox": [[0.104, 0.936, 0.286, 0.945]]}]}, "tags": ["text"], "label": "O", "web_segment_id": 11, "global_sentence_id": 94, "edu_l1_label": "EDU_O"}, {"txt": "header", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 3500000, "bbox": [[0.081, 0.049, 0.896, 0.062]]}]}, "tags": ["header"], "label": "O", "web_segment_id": 1, "global_sentence_id": 95, "edu_l1_label": "EDU_O"}, {"txt": "meaning that the results from the first pass of the LLM are deemed correct and appropriate for the vast major-ity of the protocol sections written.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 3500000, "bbox": [[0.082, 0.092, 0.474, 0.099], [0.081, 0.106, 0.479, 0.113], [0.086, 0.121, 0.339, 0.128]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 2, "global_sentence_id": 96, "edu_l1_label": "IOS"}, {"txt": "However, for clini-cal thinking and logic, the off-the-shelf LLM scores poorly (assessment score just over 40%),meaning that recommendations from the off-the-shelf LLM often do not follow the latest regulatory guidance or contain other errors.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 3500000, "bbox": [[0.339, 0.121, 0.478, 0.128], [0.082, 0.139, 0.478, 0.144], [0.082, 0.153, 0.374, 0.159], [0.374, 0.157, 0.478, 0.159], [0.082, 0.168, 0.478, 0.174], [0.082, 0.183, 0.477, 0.189], [0.082, 0.198, 0.175, 0.205]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 2, "global_sentence_id": 97, "edu_l1_label": "IOS"}, {"txt": "Since the LLM used for this analysis6 does not natively source references, assessing transparency and references is not possible (therefore, no score for this dimension).", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 3500000, "bbox": [[0.175, 0.198, 0.478, 0.205], [0.082, 0.213, 0.474, 0.221], [0.082, 0.229, 0.479, 0.235], [0.082, 0.242, 0.196, 0.251]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 2, "global_sentence_id": 98, "edu_l1_label": "IOS"}, {"txt": "As an illustrative example when we asked the algo-rithm to draft a Phase 3 protocol for tuberculosis the off-the-shelf LLM suggested in the eligibility section to exclude patients with human immunodeficiency virus (HIV)/acquired immunodeficiency syndrome,diabetes,liver disease, and kidney disease.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 3600000, "bbox": [[0.103, 0.257, 0.476, 0.264], [0.083, 0.273, 0.478, 0.281], [0.082, 0.29, 0.477, 0.296], [0.081, 0.305, 0.474, 0.311], [0.082, 0.318, 0.48, 0.328], [0.083, 0.334, 0.33, 0.342]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 3, "global_sentence_id": 99, "edu_l1_label": "IOS"}, {"txt": "This contrasts with regulatory guidance documents which state that “Sponsors should include in trials [...], subjects with renal insufficiency, diabetes mellitus, and subjects with hepatic impairment, if feasible.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 3600000, "bbox": [[0.33, 0.334, 0.477, 0.342], [0.082, 0.351, 0.478, 0.357], [0.083, 0.363, 0.478, 0.373], [0.082, 0.382, 0.478, 0.388], [0.083, 0.395, 0.319, 0.403]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 3, "global_sentence_id": 100, "edu_l1_label": "IOS"}, {"txt": "Because of the high incidence of tuberculosis in patients coinfected with HIV, subjects with HIV should be included in trials.15", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 3600000, "bbox": [[0.319, 0.395, 0.477, 0.404], [0.082, 0.412, 0.476, 0.418], [0.083, 0.424, 0.474, 0.429]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 3, "global_sentence_id": 101, "edu_l1_label": "IOS"}, {"txt": "The output of the RAG-augmented LLM (Figure 2)shows high content relevance and suitability and medical and clinical terminology, comparable to the off-the-shelf LLM.However, the RAG-augmented LLM substan-tially outperforms the off-the-shelf LLM in terms of clinical thinking and logic, where the output of the RAG-augmented LLM scores approximately twice as high as the off-the-shelf LLM.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 3700000, "bbox": [[0.103, 0.44, 0.479, 0.45], [0.081, 0.457, 0.479, 0.465], [0.082, 0.474, 0.479, 0.479], [0.082, 0.486, 0.479, 0.495], [0.083, 0.502, 0.479, 0.509], [0.082, 0.518, 0.478, 0.526], [0.084, 0.532, 0.478, 0.541], [0.083, 0.547, 0.306, 0.555]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 4, "global_sentence_id": 102, "edu_l1_label": "IOS"}, {"txt": "Regarding transparency and references, the RAG-augmented LLM (by design)includes references, which we show to be correct and appropriate nearly 80% of the time.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 3700000, "bbox": [[0.306, 0.547, 0.474, 0.557], [0.082, 0.565, 0.478, 0.571], [0.082, 0.579, 0.476, 0.587], [0.082, 0.595, 0.34, 0.601]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 4, "global_sentence_id": 103, "edu_l1_label": "IOS"}, {"txt": "As detailed in Supplementary Table 2, those trends were similar across different protocol sections, with a marked improvement in clinical thinking and logic, and similar scores in medical and clinical terminology and content relevance and suitability (the difference in scores for endpoints was not statistically significant).Transparency and references scores of the RAG-aug-mented LLM were comparable for both protocol sec-tions.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 3800000, "bbox": [[0.103, 0.608, 0.479, 0.617], [0.082, 0.625, 0.477, 0.632], [0.082, 0.641, 0.476, 0.648], [0.082, 0.656, 0.477, 0.663], [0.082, 0.672, 0.477, 0.678], [0.083, 0.686, 0.48, 0.697], [0.085, 0.7, 0.479, 0.706], [0.083, 0.717, 0.476, 0.724], [0.082, 0.732, 0.127, 0.739]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 5, "global_sentence_id": 104, "edu_l1_label": "IOS"}, {"txt": "Please note that if Bonferroni correction is applied (to account for multiple comparisons), only those related to clinical thinking and logic survive.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 3800000, "bbox": [[0.127, 0.732, 0.478, 0.739], [0.082, 0.747, 0.476, 0.754], [0.082, 0.763, 0.453, 0.77]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 5, "global_sentence_id": 105, "edu_l1_label": "IOS"}, {"txt": "On that dimension, we observed a large difference between off-the shelf LLM and RAG-augmented LLM for the endpoints section (score uplift from ~50% to ~73%)and even more pronounced for the eligibility criteria section (score uplift from ~33% to~86%).", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 3800000, "bbox": [[0.453, 0.763, 0.478, 0.77], [0.082, 0.776, 0.478, 0.785], [0.082, 0.793, 0.478, 0.8], [0.082, 0.809, 0.479, 0.816], [0.082, 0.825, 0.478, 0.831], [0.082, 0.839, 0.395, 0.846]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 5, "global_sentence_id": 106, "edu_l1_label": "IOS"}, {"txt": "Discussion", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 3900000, "bbox": [[0.082, 0.879, 0.18, 0.89]]}]}, "tags": ["title"], "label": "title1", "web_segment_id": 6, "global_sentence_id": 107, "edu_l1_label": "BOS"}, {"txt": "Across both endpoints and eligibility criteria sections,we find that the off-the-shelf LLM produces seemingly well-written content, as reflected by high scores in", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4000000, "bbox": [[0.082, 0.904, 0.48, 0.914], [0.082, 0.923, 0.476, 0.929], [0.083, 0.938, 0.477, 0.945]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 7, "global_sentence_id": 108, "edu_l1_label": "IOS"}, {"txt": "content relevance and suitability and medical and clini-cal terminology.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4000000, "bbox": [[0.499, 0.092, 0.893, 0.098], [0.499, 0.106, 0.62, 0.113]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 8, "global_sentence_id": 109, "edu_l1_label": "IOS"}, {"txt": "Closer investigation of the protocol's specifics, however, reveals important deviations from regulatory guidance (such as the above shown exam-ple), captured by low clinical thinking and logic scores.Given the critical importance of following regulation,our findings could present a challenge to the use of some off-the-shelf LLMs in the context of clinical trials and may limit their adoption in trial-related document writing.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4000000, "bbox": [[0.62, 0.106, 0.894, 0.113], [0.499, 0.122, 0.891, 0.128], [0.499, 0.137, 0.895, 0.143], [0.499, 0.152, 0.896, 0.158], [0.5, 0.165, 0.895, 0.174], [0.499, 0.182, 0.895, 0.188], [0.498, 0.196, 0.895, 0.203], [0.499, 0.212, 0.895, 0.219], [0.5, 0.228, 0.556, 0.234]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 8, "global_sentence_id": 110, "edu_l1_label": "IOS"}, {"txt": "Another major limitation of some off-the-shelf LLMs is their lack of proper referencing.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4000000, "bbox": [[0.556, 0.228, 0.896, 0.234], [0.5, 0.239, 0.791, 0.249]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 8, "global_sentence_id": 111, "edu_l1_label": "IOS"}, {"txt": "To address these challenges, we explored alternative approaches of using LLMs, specifically RAG which has emerged as a promising methodology for incorpor-ating knowledge from external databases.13,14 RAG involves providing the LLM with external sources of knowledge, to supplement the model's internal repre-sentation of information.14 Asa result of the RAG methodology, the LLM is primarily used not for its memorized knowledge, but instead for its ability to read,synthesize,and evaluate information provided to it.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4100000, "bbox": [[0.52, 0.255, 0.894, 0.264], [0.498, 0.273, 0.892, 0.279], [0.499, 0.287, 0.894, 0.293], [0.499, 0.303, 0.846, 0.304], [0.856, 0.3, 0.893, 0.31], [0.499, 0.317, 0.895, 0.325], [0.499, 0.331, 0.894, 0.339], [0.499, 0.349, 0.727, 0.355], [0.736, 0.349, 0.893, 0.355], [0.5, 0.363, 0.894, 0.37], [0.5, 0.379, 0.894, 0.385], [0.499, 0.394, 0.894, 0.4], [0.499, 0.408, 0.516, 0.416]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 9, "global_sentence_id": 112, "edu_l1_label": "IOS"}, {"txt": "In our assessment, the use of RAG augmentation produced high scores for both clinical thinking and logic and for transparency and references, demonstrating the strength of LLMs and their ability to go beyond writ-ing tasks and reason on novel information provided via RAG.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4200000, "bbox": [[0.52, 0.421, 0.894, 0.43], [0.499, 0.439, 0.894, 0.446], [0.499, 0.454, 0.894, 0.461], [0.498, 0.469, 0.894, 0.473], [0.5, 0.484, 0.893, 0.491], [0.499, 0.499, 0.576, 0.506]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 10, "global_sentence_id": 113, "edu_l1_label": "IOS"}, {"txt": "The improvement obtained by the RAG-augmented LLM for clinical thinking and logic wwas particularly high for eligibility criteria.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4200000, "bbox": [[0.576, 0.499, 0.894, 0.506], [0.499, 0.514, 0.894, 0.521], [0.5, 0.53, 0.793, 0.536]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 10, "global_sentence_id": 114, "edu_l1_label": "IOS"}, {"txt": "This demon-strates the remarkable ability of RAG-augmented LLMs to exploit vector store databases to find relevant pieces of information within large and complex source documents.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4200000, "bbox": [[0.793, 0.53, 0.894, 0.537], [0.499, 0.545, 0.893, 0.551], [0.5, 0.557, 0.895, 0.566], [0.5, 0.576, 0.894, 0.582], [0.499, 0.588, 0.581, 0.597]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 10, "global_sentence_id": 115, "edu_l1_label": "IOS"}, {"txt": "While the results shown in Figure 2 are intriguing,it is important to acknowledge a number of limitations of this work:", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4300000, "bbox": [[0.52, 0.603, 0.893, 0.612], [0.5, 0.619, 0.895, 0.627], [0.499, 0.634, 0.573, 0.642]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 11, "global_sentence_id": 116, "edu_l1_label": "IOS"}, {"txt": " First, the evaluation framework we report is a mixture of quantitative scores (clinical thinking and logic; and transparency and references) and qualita-tive scores (medical and clinical terminology; and content relevance and suitability).", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4300000, "bbox": [[0.573, 0.634, 0.894, 0.642], [0.499, 0.651, 0.894, 0.658], [0.499, 0.666, 0.894, 0.671], [0.499, 0.679, 0.894, 0.687], [0.499, 0.696, 0.744, 0.702]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 11, "global_sentence_id": 117, "edu_l1_label": "IOS"}, {"txt": "The framework thus addresses-at least in part-one of the challenges of assessing clinical trial-related writing,namely, the lack of an objective \"ground truth.”Going forward,it may be helpful to evolve our framework and make it fully quantitative.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4300000, "bbox": [[0.744, 0.696, 0.895, 0.703], [0.499, 0.711, 0.895, 0.717], [0.499, 0.727, 0.893, 0.733], [0.5, 0.741, 0.894, 0.75], [0.5, 0.754, 0.893, 0.766], [0.499, 0.771, 0.593, 0.778]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 11, "global_sentence_id": 118, "edu_l1_label": "IOS"}, {"txt": "Second, the evaluation framework is,by design, relatively general.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4300000, "bbox": [[0.593, 0.771, 0.894, 0.781], [0.5, 0.784, 0.682, 0.793]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 11, "global_sentence_id": 119, "edu_l1_label": "IOS"}, {"txt": "In our analysis, we cover spe-cific diseases and trial phases but did not tailor it to particular treatments.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4300000, "bbox": [[0.682, 0.784, 0.894, 0.793], [0.499, 0.801, 0.893, 0.808], [0.5, 0.817, 0.658, 0.823]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 11, "global_sentence_id": 120, "edu_l1_label": "IOS"}, {"txt": "In practice, this means that the RAG-augmented LLM approach can generate high-quality first draft versions of documents, which would then require further refinement to align with the treat-ment or therapy being studied.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4300000, "bbox": [[0.658, 0.817, 0.894, 0.824], [0.5, 0.829, 0.894, 0.839], [0.499, 0.848, 0.893, 0.853], [0.499, 0.861, 0.894, 0.866], [0.5, 0.877, 0.73, 0.884]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 11, "global_sentence_id": 121, "edu_l1_label": "IOS"}, {"txt": "Third, the assessment we report evaluates document sections independently of one another.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4300000, "bbox": [[0.73, 0.877, 0.894, 0.884], [0.5, 0.892, 0.893, 0.901], [0.499, 0.907, 0.615, 0.914]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 11, "global_sentence_id": 122, "edu_l1_label": "IOS"}, {"txt": "As a result, it does not capture inter-relationships across the full document.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4300000, "bbox": [[0.615, 0.907, 0.893, 0.913], [0.499, 0.923, 0.788, 0.929]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 11, "global_sentence_id": 123, "edu_l1_label": "IOS"}, {"txt": "This could be important in complex diseases where the endpoints", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4300000, "bbox": [[0.788, 0.923, 0.894, 0.929], [0.499, 0.935, 0.893, 0.945]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 11, "global_sentence_id": 124, "edu_l1_label": "IOS"}, {"txt": "header", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4400000, "bbox": [[0.102, 0.049, 0.918, 0.062]]}]}, "tags": ["header"], "label": "O", "web_segment_id": 1, "global_sentence_id": 125, "edu_l1_label": "EDU_O"}, {"txt": "should depend on the target group, defined by eligibil-ity criteria.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4400000, "bbox": [[0.102, 0.089, 0.503, 0.282]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 2, "global_sentence_id": 126, "edu_l1_label": "IOS"}, {"txt": "In such situations, LLM-based writing approaches would be most valuable if they were able to jointly generate and evaluate endpoints and eligibility criteria.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4400000, "bbox": [[0.102, 0.089, 0.503, 0.282]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 2, "global_sentence_id": 127, "edu_l1_label": "IOS"}, {"txt": "One recent approach for this could be to use agent self-reflection,$16$iterating on all protocol sections.Fourth, our assessment looks at only one LLM,namely, GPT-4.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4400000, "bbox": [[0.102, 0.089, 0.503, 0.282]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 2, "global_sentence_id": 128, "edu_l1_label": "IOS"}, {"txt": "This was motivated by the model's popularity and widespread use today.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4400000, "bbox": [[0.102, 0.089, 0.503, 0.282]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 2, "global_sentence_id": 129, "edu_l1_label": "IOS"}, {"txt": "Going forward,additional LLMs (such as Claude 3 Opus¹7)will need to be studied for comparison.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4400000, "bbox": [[0.102, 0.089, 0.503, 0.282]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 2, "global_sentence_id": 130, "edu_l1_label": "IOS"}, {"txt": "Given the recent progress in the field, we expect newer LLMs to have improved performance characteristics.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4400000, "bbox": [[0.102, 0.089, 0.503, 0.282]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 2, "global_sentence_id": 131, "edu_l1_label": "IOS"}, {"txt": "In summary, our results suggest that hybrid LLM architectures such as agent-based RAG methodology we used offer strong potential for GenAl-powered clini-cal trial-related writing, covering potentially a wide variety of documents.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4500000, "bbox": [[0.101, 0.283, 0.503, 0.583]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 3, "global_sentence_id": 132, "edu_l1_label": "IOS"}, {"txt": "This is exciting, since it addresses several major bottlenecks of drug development.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4500000, "bbox": [[0.101, 0.283, 0.503, 0.583]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 3, "global_sentence_id": 133, "edu_l1_label": "IOS"}, {"txt": "Indeed,when we applied the RAG-augmented LLM approach in the context of recent clinical trials, we observed dra-matic acceleration.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4500000, "bbox": [[0.101, 0.283, 0.503, 0.583]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 3, "global_sentence_id": 134, "edu_l1_label": "IOS"}, {"txt": "For writing tasks, such as protocols or clinical study reports, the time to generate first draft versions of documents is typically reduced from days or weeks (in the case of fully-human writing), to min-utes (when uusing a RAG-augmented LLM).", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4500000, "bbox": [[0.101, 0.283, 0.503, 0.583]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 3, "global_sentence_id": 135, "edu_l1_label": "IOS"}, {"txt": "For the end-to-end document creation process, which normally consists of multiple cycles of writing and review,we observe time reductions of $25\\%-50\\%$ or more, depend-ing on which document is being created.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4500000, "bbox": [[0.101, 0.283, 0.503, 0.583]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 3, "global_sentence_id": 136, "edu_l1_label": "IOS"}, {"txt": "The reason why the time reduction of the end-to-end process is somewhat smaller than for writing alone is because review by human experts is always required.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4500000, "bbox": [[0.101, 0.283, 0.503, 0.583]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 3, "global_sentence_id": 137, "edu_l1_label": "IOS"}, {"txt": "Beyond the writing abilities of LLMs in clinical trials,which our work demonstrates, there are a num-ber of practical1 considerations which pharmaceutical companies and other trial sponsors will need to address.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4600000, "bbox": [[0.126, 0.588, 0.499, 0.597], [0.103, 0.605, 0.499, 0.612], [0.104, 0.618, 0.499, 0.627], [0.103, 0.635, 0.498, 0.642], [0.103, 0.65, 0.165, 0.657]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 4, "global_sentence_id": 138, "edu_l1_label": "IOS"}, {"txt": "First and most importantly, there are ques-tions about the ethical use of LLMs and other GenAI tools in the context of clinical trials.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4600000, "bbox": [[0.165, 0.65, 0.499, 0.656], [0.104, 0.664, 0.499, 0.673], [0.104, 0.679, 0.362, 0.688]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 4, "global_sentence_id": 139, "edu_l1_label": "IOS"}, {"txt": "If sponsors wish to utilize these tools in the design of clinical trials and the writing of documents, it is imperative that there be responsible human oversight.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4600000, "bbox": [[0.362, 0.679, 0.499, 0.687], [0.105, 0.695, 0.5, 0.703], [0.104, 0.711, 0.499, 0.718], [0.103, 0.726, 0.317, 0.732]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 4, "global_sentence_id": 140, "edu_l1_label": "IOS"}, {"txt": "Clinical trial experts and medical writers will need to be “in the loop\" to ensure that trials are designed and executed safely and follow-ing all applicable rules and guidelines.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4600000, "bbox": [[0.317, 0.726, 0.498, 0.733], [0.104, 0.741, 0.5, 0.748], [0.104, 0.755, 0.499, 0.762], [0.104, 0.771, 0.395, 0.778]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 4, "global_sentence_id": 141, "edu_l1_label": "IOS"}, {"txt": "Second,there are regulatory questions.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4600000, "bbox": [[0.395, 0.771, 0.499, 0.778], [0.103, 0.787, 0.283, 0.793]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 4, "global_sentence_id": 142, "edu_l1_label": "IOS"}, {"txt": "The US FDA and other agen-cies have outlined their plans to regulate artificial intel-ligence in medical products, including building relevant infrastructure and technical expertise.18 As the regula-tory framework evolves, sponsors of clinical trials will likely need to adapt their LLM and other tools in clini-cal trial writing and other processes.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4600000, "bbox": [[0.283, 0.787, 0.499, 0.793], [0.103, 0.802, 0.5, 0.808], [0.105, 0.814, 0.499, 0.824], [0.103, 0.832, 0.499, 0.838], [0.104, 0.846, 0.5, 0.854], [0.104, 0.86, 0.5, 0.868], [0.103, 0.877, 0.378, 0.884]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 4, "global_sentence_id": 143, "edu_l1_label": "IOS"}, {"txt": "Third, there are talent and capability considerations.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4600000, "bbox": [[0.378, 0.877, 0.499, 0.884], [0.104, 0.891, 0.365, 0.899]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 4, "global_sentence_id": 144, "edu_l1_label": "IOS"}, {"txt": "Hiring and retain-ing suitable expertise, ideally combining clinical trial knowledge and GenAI technical experience, is critical for these efforts to succeed.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4600000, "bbox": [[0.365, 0.891, 0.499, 0.898], [0.104, 0.906, 0.499, 0.914], [0.104, 0.921, 0.499, 0.929], [0.104, 0.935, 0.325, 0.945]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 4, "global_sentence_id": 145, "edu_l1_label": "IOS"}, {"txt": "A recent white paper,", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4600000, "bbox": [[0.325, 0.935, 0.5, 0.945]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 4, "global_sentence_id": 146, "edu_l1_label": "IOS"}, {"txt": "jointly authored by major pharmaceutical companies,0highlights talent as a major challenge for the industry.Fourth, there are questions regarding technical readi-ness.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4600000, "bbox": [[0.521, 0.089, 0.914, 0.101], [0.906, 0.101, 0.912, 0.107], [0.521, 0.104, 0.904, 0.113], [0.521, 0.119, 0.916, 0.128], [0.521, 0.138, 0.557, 0.144]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 5, "global_sentence_id": 147, "edu_l1_label": "IOS"}, {"txt": "In recent years, many pharmaceutical companies and other trial sponsors have made major investments in their data analytics platforms and in data partner-ships.19 However, from the work reported here, we learned that configuring these technologies, and ingest-ing, integrating, and analyzing the required data sources is often challenging.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4600000, "bbox": [[0.557, 0.138, 0.916, 0.144], [0.521, 0.153, 0.916, 0.159], [0.52, 0.166, 0.915, 0.171], [0.52, 0.183, 0.915, 0.19], [0.521, 0.196, 0.915, 0.202], [0.52, 0.213, 0.915, 0.22], [0.52, 0.229, 0.72, 0.236]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 5, "global_sentence_id": 148, "edu_l1_label": "IOS"}, {"txt": "Despite these challenges, our experience of starting to deploy LLMs in a number of real-life settings sug-gests strong potential of accelerating and improving clinical trial-related writing.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4700000, "bbox": [[0.542, 0.241, 0.915, 0.252], [0.52, 0.259, 0.915, 0.263], [0.521, 0.275, 0.915, 0.282], [0.52, 0.289, 0.733, 0.296]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 6, "global_sentence_id": 149, "edu_l1_label": "IOS"}, {"txt": "These benefits typically require a strong collaboration between medical writers,clinical researchers, data scientists, and data engineers.Organizations that achieve this cross-functional colla-boration are already beginning to reap significant accel-eration gains.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4700000, "bbox": [[0.733, 0.289, 0.915, 0.298], [0.521, 0.305, 0.915, 0.312], [0.52, 0.319, 0.916, 0.327], [0.521, 0.333, 0.916, 0.343], [0.521, 0.349, 0.914, 0.357], [0.52, 0.366, 0.62, 0.373]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 6, "global_sentence_id": 150, "edu_l1_label": "IOS"}, {"txt": "Going forward, we expect these benefits to increase even further.Over time, we therefore expect that sponsors of clinical trials will adopt the LLM tech-nology in their clinical and other writing tasks.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4700000, "bbox": [[0.62, 0.366, 0.917, 0.373], [0.521, 0.38, 0.916, 0.389], [0.521, 0.395, 0.916, 0.402], [0.521, 0.412, 0.853, 0.418]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 6, "global_sentence_id": 151, "edu_l1_label": "IOS"}, {"txt": "Acknowledgements", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4800000, "bbox": [[0.52, 0.453, 0.673, 0.462]]}]}, "tags": ["title"], "label": "title1", "web_segment_id": 7, "global_sentence_id": 152, "edu_l1_label": "EDU_O"}, {"txt": "The authors thank their colleagues Dr Jennifer Griffin and Dr Souparno Bhattacharya for their assistance in conducting this assessment.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 4900000, "bbox": [[0.521, 0.474, 0.916, 0.481], [0.521, 0.487, 0.916, 0.496], [0.521, 0.503, 0.593, 0.509]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 8, "global_sentence_id": 153, "edu_l1_label": "EDU_O"}, {"txt": "Author contributions", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 5000000, "bbox": [[0.52, 0.543, 0.686, 0.552]]}]}, "tags": ["title"], "label": "title1", "web_segment_id": 9, "global_sentence_id": 154, "edu_l1_label": "EDU_O"}, {"txt": "N.M.:", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 5100000, "bbox": [[0.521, 0.563, 0.563, 0.576]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 10, "global_sentence_id": 155, "edu_l1_label": "EDU_O"}, {"txt": " Methodology, Data processing, LLM and other analy-sis, and Manuscript writing, review, and editing.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 5100000, "bbox": [[0.563, 0.563, 0.916, 0.572], [0.52, 0.579, 0.827, 0.586]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 10, "global_sentence_id": 156, "edu_l1_label": "EDU_O"}, {"txt": "C.v.L.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 5200000, "bbox": [[0.521, 0.591, 0.565, 0.599]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 11, "global_sentence_id": 157, "edu_l1_label": "EDU_O"}, {"txt": ": Methodology, Data processing, LLM and other anal-ysis, and Manuscript writing, review, and editing.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 5200000, "bbox": [[0.565, 0.591, 0.916, 0.598], [0.521, 0.607, 0.838, 0.613]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 11, "global_sentence_id": 158, "edu_l1_label": "EDU_O"}, {"txt": "G.R.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 5300000, "bbox": [[0.521, 0.618, 0.558, 0.626]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 12, "global_sentence_id": 159, "edu_l1_label": "EDU_O"}, {"txt": ": Methodology, Data processing, LLM and other analy-sis,and Manuscript review and editing.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 5300000, "bbox": [[0.558, 0.618, 0.916, 0.625], [0.521, 0.634, 0.773, 0.64]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 12, "global_sentence_id": 160, "edu_l1_label": "EDU_O"}, {"txt": "I.E.-M.:", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 5400000, "bbox": [[0.521, 0.646, 0.575, 0.651]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 13, "global_sentence_id": 161, "edu_l1_label": "EDU_O"}, {"txt": " Methodology, Data processing,and LLM and other analysis.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 5400000, "bbox": [[0.575, 0.646, 0.916, 0.655], [0.52, 0.662, 0.577, 0.669]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 13, "global_sentence_id": 162, "edu_l1_label": "EDU_O"}, {"txt": "C.M.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 5500000, "bbox": [[0.521, 0.674, 0.558, 0.683]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 14, "global_sentence_id": 163, "edu_l1_label": "EDU_O"}, {"txt": ": Methodology, LLM and other analysis, and Manuscript writing,review,and editing.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 5500000, "bbox": [[0.558, 0.674, 0.915, 0.683], [0.521, 0.688, 0.778, 0.696]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 14, "global_sentence_id": 164, "edu_l1_label": "EDU_O"}, {"txt": "All authors contributed to the article and approved the sub-mitted version.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 5600000, "bbox": [[0.521, 0.702, 0.914, 0.71], [0.521, 0.718, 0.617, 0.724]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 15, "global_sentence_id": 165, "edu_l1_label": "EDU_O"}, {"txt": "Declaration of conflicting interests", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 5700000, "bbox": [[0.521, 0.75, 0.788, 0.759]]}]}, "tags": ["title"], "label": "title1", "web_segment_id": 16, "global_sentence_id": 166, "edu_l1_label": "EDU_O"}, {"txt": "The author(s) declared the following potential conflicts of interest with respect to the research, authorship, and/or publi-cation of this article:", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 5800000, "bbox": [[0.521, 0.771, 0.917, 0.778], [0.521, 0.787, 0.916, 0.791], [0.521, 0.8, 0.655, 0.807]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 17, "global_sentence_id": 167, "edu_l1_label": "EDU_O"}, {"txt": " The authors of this article are employees of The Boston Consulting Group (BCG), a management con-sultancy that works with the world's leading companies.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 5800000, "bbox": [[0.655, 0.8, 0.917, 0.807], [0.521, 0.814, 0.915, 0.82], [0.52, 0.828, 0.879, 0.834]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 17, "global_sentence_id": 168, "edu_l1_label": "EDU_O"}, {"txt": "Funding", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 5900000, "bbox": [[0.521, 0.86, 0.584, 0.872]]}]}, "tags": ["title"], "label": "title1", "web_segment_id": 18, "global_sentence_id": 169, "edu_l1_label": "EDU_O"}, {"txt": "The author(s) disclosed receipt of the following financial support for the research, authorship, and/or publication of this article:", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 6000000, "bbox": [[0.521, 0.881, 0.916, 0.889], [0.52, 0.897, 0.916, 0.903], [0.52, 0.909, 0.593, 0.917]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 19, "global_sentence_id": 170, "edu_l1_label": "EDU_O"}, {"txt": " The research for this article was funded by BCG s Health Care practice and by BCG X, the firm's in-house data science unit.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 6000000, "bbox": [[0.593, 0.909, 0.916, 0.917], [0.521, 0.922, 0.915, 0.93], [0.52, 0.938, 0.6, 0.945]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 19, "global_sentence_id": 171, "edu_l1_label": "EDU_O"}, {"txt": "header", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 6100000, "bbox": [[0.081, 0.049, 0.897, 0.064]]}]}, "tags": ["header"], "label": "O", "web_segment_id": 1, "global_sentence_id": 172, "edu_l1_label": "EDU_O"}, {"txt": "ORCID iD", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 6200000, "bbox": [[0.082, 0.088, 0.163, 0.098]]}]}, "tags": ["title"], "label": "title1", "web_segment_id": 2, "global_sentence_id": 173, "edu_l1_label": "EDU_O"}, {"txt": "Christoph Meier D https://orcid.org/0009-0005-6323-8504", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 6300000, "bbox": [[0.082, 0.108, 0.458, 0.116]]}]}, "tags": ["text"], "label": "O", "web_segment_id": 3, "global_sentence_id": 174, "edu_l1_label": "EDU_O"}, {"txt": "Supplemental material", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 6400000, "bbox": [[0.082, 0.144, 0.264, 0.153]]}]}, "tags": ["title"], "label": "title1", "web_segment_id": 4, "global_sentence_id": 175, "edu_l1_label": "EDU_O"}, {"txt": "Supplemental material for this article is available online.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 6500000, "bbox": [[0.082, 0.164, 0.443, 0.171]]}]}, "tags": ["text"], "label": "content", "web_segment_id": 5, "global_sentence_id": 176, "edu_l1_label": "EDU_O"}, {"txt": "References", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 6600000, "bbox": [[0.082, 0.198, 0.169, 0.208]]}]}, "tags": ["title"], "label": "reference", "web_segment_id": 6, "global_sentence_id": 177, "edu_l1_label": "EDU_O"}, {"txt": "1.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 6700000, "bbox": [[0.092, 0.218, 0.104, 0.227]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 7, "global_sentence_id": 178, "edu_l1_label": "EDU_O"}, {"txt": "Sharma S. How to become a competent medical writer?Perspect Clin Res 2010; 1(1):33-37.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 6700000, "bbox": [[0.104, 0.218, 0.478, 0.227], [0.112, 0.232, 0.338, 0.241]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 7, "global_sentence_id": 179, "edu_l1_label": "EDU_O"}, {"txt": "2.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 6800000, "bbox": [[0.09, 0.246, 0.105, 0.254]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 8, "global_sentence_id": 180, "edu_l1_label": "EDU_O"}, {"txt": "Spence O, Hong K, Onwuchekwa Uba R, et al.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 6800000, "bbox": [[0.105, 0.246, 0.413, 0.254]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 8, "global_sentence_id": 181, "edu_l1_label": "EDU_O"}, {"txt": "Availabil-ity of study protocols for randomized trials published in high-impact medical journals: a cross-sectional analysis.Clin Trials 2020; 17(1):99-105.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 6800000, "bbox": [[0.413, 0.246, 0.479, 0.252], [0.111, 0.261, 0.477, 0.268], [0.111, 0.275, 0.479, 0.282], [0.111, 0.288, 0.311, 0.296]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 8, "global_sentence_id": 182, "edu_l1_label": "EDU_O"}, {"txt": "3.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 6900000, "bbox": [[0.09, 0.301, 0.105, 0.31]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 9, "global_sentence_id": 183, "edu_l1_label": "EDU_O"}, {"txt": "Landray MJ, Haynes R and Reith C. Accelerating clinical trials: time to turn words into action.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 6900000, "bbox": [[0.105, 0.301, 0.479, 0.309], [0.109, 0.317, 0.36, 0.323]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 9, "global_sentence_id": 184, "edu_l1_label": "EDU_O"}, {"txt": "Lancet 2023;402:165-168.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 6900000, "bbox": [[0.36, 0.317, 0.479, 0.323], [0.111, 0.329, 0.166, 0.338]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 9, "global_sentence_id": 185, "edu_l1_label": "EDU_O"}, {"txt": "4.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 7000000, "bbox": [[0.09, 0.343, 0.106, 0.351]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 10, "global_sentence_id": 186, "edu_l1_label": "EDU_O"}, {"txt": "Markey N, Howitt B, El-Mansouri I, et al.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 7000000, "bbox": [[0.106, 0.343, 0.39, 0.351]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 10, "global_sentence_id": 187, "edu_l1_label": "EDU_O"}, {"txt": "Clinical trials are becoming more complex: a machine learning analysis of data from over 16,000 trials.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 7000000, "bbox": [[0.39, 0.343, 0.478, 0.351], [0.11, 0.359, 0.479, 0.365], [0.11, 0.373, 0.312, 0.383]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 10, "global_sentence_id": 188, "edu_l1_label": "EDU_O"}, {"txt": "Sci Rep 2024;14:3514.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 7000000, "bbox": [[0.312, 0.373, 0.463, 0.379]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 10, "global_sentence_id": 189, "edu_l1_label": "EDU_O"}, {"txt": "5.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 7100000, "bbox": [[0.09, 0.385, 0.105, 0.392]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 11, "global_sentence_id": 190, "edu_l1_label": "EDU_O"}, {"txt": "Biever C. ChatGPT broke the turing test-the race is on for new ways to assess AI.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 7100000, "bbox": [[0.105, 0.385, 0.479, 0.393], [0.113, 0.401, 0.252, 0.407]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 11, "global_sentence_id": 191, "edu_l1_label": "EDU_O"}, {"txt": "Nature 2023;619(7971):686-689.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 7100000, "bbox": [[0.252, 0.401, 0.458, 0.407]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 11, "global_sentence_id": 192, "edu_l1_label": "EDU_O"}, {"txt": "6.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 7200000, "bbox": [[0.09, 0.412, 0.105, 0.42]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 12, "global_sentence_id": 193, "edu_l1_label": "EDU_O"}, {"txt": "OpenAI.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 7200000, "bbox": [[0.105, 0.412, 0.169, 0.42]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 12, "global_sentence_id": 194, "edu_l1_label": "EDU_O"}, {"txt": "ChatGPT.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 7200000, "bbox": [[0.169, 0.412, 0.242, 0.42]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 12, "global_sentence_id": 195, "edu_l1_label": "EDU_O"}, {"txt": "Software tool based on a large lan-guage model,2023,https://openai.com/index/chatgpt/", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 7200000, "bbox": [[0.242, 0.412, 0.478, 0.421], [0.111, 0.428, 0.457, 0.434]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 12, "global_sentence_id": 196, "edu_l1_label": "EDU_O"}, {"txt": "7.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 7300000, "bbox": [[0.09, 0.44, 0.107, 0.448]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 13, "global_sentence_id": 197, "edu_l1_label": "EDU_O"}, {"txt": "Jin Q, Wang Z, Floudas CS, et al.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 7300000, "bbox": [[0.107, 0.44, 0.339, 0.453]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 13, "global_sentence_id": 198, "edu_l1_label": "EDU_O"}, {"txt": "Matchingpatients to clinical trials with large language models.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 7300000, "bbox": [[0.339, 0.44, 0.478, 0.448], [0.11, 0.455, 0.396, 0.462]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 13, "global_sentence_id": 199, "edu_l1_label": "EDU_O"}, {"txt": "arXiv 2023;2307.15051,https://arxiv.org/pdf/2307.15051", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 7300000, "bbox": [[0.396, 0.455, 0.479, 0.463], [0.111, 0.467, 0.395, 0.476]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 13, "global_sentence_id": 200, "edu_l1_label": "EDU_O"}, {"txt": "8.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 7400000, "bbox": [[0.089, 0.482, 0.106, 0.489]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 14, "global_sentence_id": 201, "edu_l1_label": "EDU_O"}, {"txt": "Wang Z, Xiao C and Sun J. AutoTrial: prompting lan-guage models for clinical trial design.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 7400000, "bbox": [[0.106, 0.482, 0.478, 0.489], [0.11, 0.498, 0.39, 0.504]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 14, "global_sentence_id": 202, "edu_l1_label": "EDU_O"}, {"txt": "arXiv 2023;2305.11366,https://arxiv.org/pdf/2305.11366", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 7400000, "bbox": [[0.39, 0.498, 0.479, 0.504], [0.111, 0.509, 0.395, 0.517]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 14, "global_sentence_id": 203, "edu_l1_label": "EDU_O"}, {"txt": "9.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 7500000, "bbox": [[0.09, 0.523, 0.106, 0.531]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 15, "global_sentence_id": 204, "edu_l1_label": "EDU_O"}, {"txt": "Patel SB and Lam K. ChatGPT: the future of discharge summaries.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 7500000, "bbox": [[0.106, 0.523, 0.477, 0.533], [0.11, 0.539, 0.185, 0.545]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 15, "global_sentence_id": 205, "edu_l1_label": "EDU_O"}, {"txt": "Lancet Digit Health 2023; 5(3):e107-e108.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 7500000, "bbox": [[0.185, 0.539, 0.458, 0.545]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 15, "global_sentence_id": 206, "edu_l1_label": "EDU_O"}, {"txt": "10.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 7600000, "bbox": [[0.5, 0.09, 0.521, 0.098]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 16, "global_sentence_id": 207, "edu_l1_label": "EDU_O"}, {"txt": "Huynh LM, Bonebrake BT, Schultis K, et al.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 7600000, "bbox": [[0.521, 0.09, 0.823, 0.098]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 16, "global_sentence_id": 208, "edu_l1_label": "EDU_O"}, {"txt": "New artifi-cial intelligence ChatGPT performs poorly on the 2022self-assessment study program for urology.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 7600000, "bbox": [[0.823, 0.09, 0.895, 0.096], [0.526, 0.106, 0.893, 0.111], [0.528, 0.119, 0.819, 0.126]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 16, "global_sentence_id": 209, "edu_l1_label": "EDU_O"}, {"txt": "Urol Pract 2023;10(4):409-415.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 7600000, "bbox": [[0.819, 0.119, 0.896, 0.126], [0.527, 0.132, 0.662, 0.14]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 16, "global_sentence_id": 210, "edu_l1_label": "EDU_O"}, {"txt": "11.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 7700000, "bbox": [[0.5, 0.146, 0.521, 0.154]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 17, "global_sentence_id": 211, "edu_l1_label": "EDU_O"}, {"txt": "Ghim JL and Ahn S. Transforming clinical trials: the emerging roles of large language models.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 7700000, "bbox": [[0.521, 0.146, 0.894, 0.154], [0.526, 0.162, 0.814, 0.168]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 17, "global_sentence_id": 212, "edu_l1_label": "EDU_O"}, {"txt": "Transl Clin Pharmacol 2023; 31(3):131-138.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 7700000, "bbox": [[0.814, 0.162, 0.894, 0.168], [0.528, 0.174, 0.735, 0.181]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 17, "global_sentence_id": 213, "edu_l1_label": "EDU_O"}, {"txt": "12.OpenAI.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 7800000, "bbox": [[0.5, 0.188, 0.587, 0.196]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 18, "global_sentence_id": 214, "edu_l1_label": "EDU_O"}, {"txt": "GPT-4 technical report.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 7800000, "bbox": [[0.587, 0.188, 0.766, 0.196]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 18, "global_sentence_id": 215, "edu_l1_label": "EDU_O"}, {"txt": "arXiv 2023; abs/2303.08774,https://arxiv.org/pdf/2303.08774", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 7800000, "bbox": [[0.766, 0.188, 0.896, 0.195], [0.526, 0.202, 0.813, 0.21]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 18, "global_sentence_id": 216, "edu_l1_label": "EDU_O"}, {"txt": "13.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 7900000, "bbox": [[0.5, 0.216, 0.52, 0.224]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 19, "global_sentence_id": 217, "edu_l1_label": "EDU_O"}, {"txt": "Gao Y,Xiong Y,Gao X,et al.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 7900000, "bbox": [[0.52, 0.216, 0.727, 0.224]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 19, "global_sentence_id": 218, "edu_l1_label": "EDU_O"}, {"txt": "Retrieval-augmented gen-eration for large language models: a survey.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 7900000, "bbox": [[0.727, 0.216, 0.894, 0.224], [0.526, 0.232, 0.817, 0.237]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 19, "global_sentence_id": 219, "edu_l1_label": "EDU_O"}, {"txt": "arXiv 2023;2312.10997,https://arxiv.org/pdf/2312.10997", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 7900000, "bbox": [[0.817, 0.232, 0.896, 0.239], [0.527, 0.243, 0.812, 0.251]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 19, "global_sentence_id": 220, "edu_l1_label": "EDU_O"}, {"txt": "14.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 8000000, "bbox": [[0.5, 0.257, 0.521, 0.266]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 20, "global_sentence_id": 221, "edu_l1_label": "EDU_O"}, {"txt": "Lewis P, Perez E, Piktus A, et al.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 8000000, "bbox": [[0.521, 0.257, 0.757, 0.266]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 20, "global_sentence_id": 222, "edu_l1_label": "EDU_O"}, {"txt": "Retrieval-augmented generation for knowledge-intensive NLP tasks.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 8000000, "bbox": [[0.757, 0.257, 0.893, 0.265], [0.527, 0.273, 0.846, 0.279]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 20, "global_sentence_id": 223, "edu_l1_label": "EDU_O"}, {"txt": "Neural Inf Process Syst 2020;793: 9459-9474.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 8000000, "bbox": [[0.846, 0.273, 0.898, 0.28], [0.527, 0.286, 0.773, 0.294]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 20, "global_sentence_id": 224, "edu_l1_label": "EDU_O"}, {"txt": "15.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 8100000, "bbox": [[0.5, 0.299, 0.521, 0.307]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 21, "global_sentence_id": 225, "edu_l1_label": "EDU_O"}, {"txt": "Department of Health and Human Services, Food and Drug Administration and Center for Drug Evaluation and Research (CDER).", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 8100000, "bbox": [[0.521, 0.299, 0.891, 0.308], [0.527, 0.314, 0.893, 0.321], [0.526, 0.329, 0.679, 0.335]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 21, "global_sentence_id": 226, "edu_l1_label": "EDU_O"}, {"txt": "Pulmonary tuberculosis: develop-ing drugs for treatment, 2022,https://www.fda.gov/media/87194/download", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 8100000, "bbox": [[0.679, 0.329, 0.893, 0.336], [0.526, 0.341, 0.894, 0.351], [0.527, 0.358, 0.676, 0.363]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 21, "global_sentence_id": 227, "edu_l1_label": "EDU_O"}, {"txt": "16.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 8200000, "bbox": [[0.5, 0.37, 0.521, 0.378]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 22, "global_sentence_id": 228, "edu_l1_label": "EDU_O"}, {"txt": "Renze M and Guven E. Self-reflection in LLM agents:effects on problem-solving performance.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 8200000, "bbox": [[0.521, 0.37, 0.896, 0.378], [0.526, 0.385, 0.808, 0.392]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 22, "global_sentence_id": 229, "edu_l1_label": "EDU_O"}, {"txt": "arXiv 2024;2405.06682,https://arxiv.org/abs/2405.06682", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 8200000, "bbox": [[0.808, 0.385, 0.896, 0.392], [0.526, 0.397, 0.813, 0.405]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 22, "global_sentence_id": 230, "edu_l1_label": "EDU_O"}, {"txt": "17.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 8300000, "bbox": [[0.5, 0.411, 0.521, 0.419]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 23, "global_sentence_id": 231, "edu_l1_label": "EDU_O"}, {"txt": "Anthropic.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 8300000, "bbox": [[0.521, 0.411, 0.599, 0.419]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 23, "global_sentence_id": 232, "edu_l1_label": "EDU_O"}, {"txt": "Claude 3 Opus [Large language model], 2023.https://www.anthropic.com/", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 8300000, "bbox": [[0.599, 0.411, 0.897, 0.419], [0.527, 0.426, 0.707, 0.434]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 23, "global_sentence_id": 233, "edu_l1_label": "EDU_O"}, {"txt": "18.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 8400000, "bbox": [[0.5, 0.44, 0.521, 0.447]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 24, "global_sentence_id": 234, "edu_l1_label": "EDU_O"}, {"txt": "US FDA.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 8400000, "bbox": [[0.521, 0.44, 0.593, 0.447]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 24, "global_sentence_id": 235, "edu_l1_label": "EDU_O"}, {"txt": "Using artificial intelligence & machine learning in the development of drug and biological products.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 8400000, "bbox": [[0.593, 0.44, 0.894, 0.449], [0.526, 0.454, 0.866, 0.461]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 24, "global_sentence_id": 236, "edu_l1_label": "EDU_O"}, {"txt": "Dis-cussion paper,2023, https://www.fda.gov/media/167973/download", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 8400000, "bbox": [[0.866, 0.454, 0.894, 0.46], [0.526, 0.47, 0.894, 0.477], [0.526, 0.482, 0.589, 0.49]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 24, "global_sentence_id": 237, "edu_l1_label": "EDU_O"}, {"txt": "19.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 8500000, "bbox": [[0.5, 0.495, 0.521, 0.503]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 25, "global_sentence_id": 238, "edu_l1_label": "EDU_O"}, {"txt": "Khan NS, Senderovitz T, Weatherall J, et al.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 8500000, "bbox": [[0.521, 0.495, 0.856, 0.503]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 25, "global_sentence_id": 239, "edu_l1_label": "EDU_O"}, {"txt": "Data science in pharmaceutical R&D: the DISRUPT-DS industry roundtable.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 8500000, "bbox": [[0.856, 0.495, 0.894, 0.504], [0.526, 0.511, 0.893, 0.517], [0.527, 0.525, 0.663, 0.531]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 25, "global_sentence_id": 240, "edu_l1_label": "EDU_O"}, {"txt": "Nat Rev Drug Discov 2024; 23(9):645-646.", "language": "english", "position": {"pdf_position": [{"page_number": 0, "shape": [1650, 1275], "position_id": 8500000, "bbox": [[0.663, 0.525, 0.896, 0.531], [0.526, 0.538, 0.585, 0.545]]}]}, "tags": ["text"], "label": "reference", "web_segment_id": 25, "global_sentence_id": 241, "edu_l1_label": "EDU_O"}], "type": "PDF"}
# From RAGs to riches: Utilizing large language models to write documents for clinical trials
## Background and aims
## Methods
## Results
## Discussion
