Upload 38 files

lo-achievement/.devcontainer/Dockerfile

@@ -0,0 +1,66 @@
+# Use the official image as a parent image.
+FROM ubuntu:22.04
+
+# Update the system
+RUN apt-get -y update && apt-get install -y bash \
+    build-essential \
+    wget \
+    make \
+    git \
+    curl \
+    sudo \
+    ca-certificates \
+    poppler-utils \
+    python3 \
+    python3-pip \
+    nodejs \
+    npm && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install python packages
+RUN python3 -m pip install --no-cache-dir --upgrade pip && \
+    python3 -m pip install --no-cache-dir torch torchvision torchaudio && \
+    python3 -m pip install --no-cache-dir \
+    transformers \
+    langchain \
+    langchain[llms] \
+    pypdf \
+    tokenizers \
+    sentencepiece \
+    openai \
+    librosa \
+    datasets \
+    jupyter \
+    jupyterlab \
+    pandas \
+    openpyxl \
+    matplotlib \
+    numpy \
+    seaborn \
+    chromadb \
+    tiktoken \
+    unstructured \
+    deeplake \
+    ipyfilechooser \
+    ipywidgets \
+    widgetsnbextension \
+    jupyter_contrib_nbextensions \
+    jupyter_nbextensions_configurator \
+    jupyterlab-git \
+    gradio \
+    streamlit \
+    nb-clean \
+    nbdime \
+    nbdev
+
+# Enable jupyter nbextension
+# RUN jupyter nbextension enable --py widgetsnbextension
+
+# Run nbdev_install_quarto
+RUN sudo nbdev_install_quarto
+
+# Indicate the Dockerfile owner
+LABEL maintainer="Charreau Bell"
+
+# Command to run on container start
+CMD [ "/bin/bash" ]

lo-achievement/.devcontainer/devcontainer.json

@@ -0,0 +1,25 @@
+{
+    "name": "CLAS Container",
+    "dockerFile": "Dockerfile",
+    //"forwardPorts": [80, 443, 8080],
+    "customizations": {
+        "vscode": {
+            "settings": {
+                "terminal.integrated.shell.linux": "/bin/bash"
+            },
+            "extensions": [
+				"ms-python.python",
+				"ms-azuretools.vscode-docker",
+				"ms-vscode-remote.remote-containers",
+                "ms-toolsai.jupyter",
+				"ms-toolsai.vscode-jupyter-cell-tags",
+				"ms-toolsai.jupyter-renderers",
+				"ms-python.vscode-pylance",
+				"ms-toolsai.jupyter-keymap"
+			]
+        }
+    }
+   // "mounts": [
+   //     "source=/host/path,target=/container/path,type=bind"
+    //],
+}

lo-achievement/.github/workflows/deploy.yaml

@@ -0,0 +1,14 @@
+name: Deploy to GitHub Pages
+
+permissions:
+  contents: write
+  pages: write
+
+on:
+  push:
+    branches: [ "main", "master" ]
+  workflow_dispatch:
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps: [uses: fastai/workflows/quarto-ghp@master]

lo-achievement/.github/workflows/test.yaml

@@ -0,0 +1,7 @@
+name: CI
+on:  [workflow_dispatch, pull_request, push]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps: [uses: fastai/workflows/nbdev-ci@master]

lo-achievement/.gitignore

@@ -0,0 +1,151 @@
+_docs/
+_proc/
+
+*.bak
+.gitattributes
+.last_checked
+.gitconfig
+*.bak
+*.log
+*~
+~*
+_tmp*
+tmp*
+tags
+*.pkg
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# dotenv
+.env
+
+# virtualenv
+.venv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+.vscode
+*.swp
+
+# osx generated files
+.DS_Store
+.DS_Store?
+.Trashes
+ehthumbs.db
+Thumbs.db
+.idea
+
+# pytest
+.pytest_cache
+
+# tools/trust-doc-nbs
+docs_src/.last_checked
+
+# symlinks to fastai
+docs_src/fastai
+tools/fastai
+
+# link checker
+checklink/cookies.txt
+
+# .gitconfig is now autogenerated
+.gitconfig
+
+# Quarto installer
+.deb
+.pkg
+
+# Quarto
+.quarto

lo-achievement/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2022, fastai
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

lo-achievement/MANIFEST.in

@@ -0,0 +1,5 @@
+include settings.ini
+include LICENSE
+include CONTRIBUTING.md
+include README.md
+recursive-exclude * __pycache__

lo-achievement/README.md

@@ -0,0 +1,81 @@
+# Classroom Learning and Assessment Suite (CLAS)
+
+The Classroom Learning and Assessment Suite is a set of AI-driven solutions that complement in-class learning with interactive study and assessment. Primarily ideated by faculty interested in leveraging AI platforms to enhance student learning, the suite creates new mechanisms for both students and faculty members in both supporting high quality learning assessments and streamlining grading of student submissions using generative AI-based platforms.
+
+## Features
+CLAS is suitable for users of diverse programming backgrounds - from no programming background to seasoned programmers. The Suite provides:
+
+### **Prompting Guides**
+
+* **[A curated prompt dictionary for self-study](https://github.com/vanderbilt-data-science/lo-achievement/wiki/Guide-to-Learning-Objective-Prompts)**: A set of prompts that have been engineering and evaluated on OpenAI GPT 3.5 and GPT4 to provide numerous types of assessments for students engaging in self-study with instructor provided materials
+* **[A curated prompt dictionary for student assessment](https://github.com/vanderbilt-data-science/lo-achievement/wiki/Bloom's-Taxonomy-Rubric-Prompts)**: Prompts targeted primarily towards instructors who want a generalized rubric for evaluating student submissions
+
+### **Interface-Driven Programs**
+No programming experience? No problem. 
+
+* **[Student self-study using instructor assigned resources](https://huggingface.co/spaces/vanderbilt-dsi/selfstudy_learningobjectives_demo)**: A point-and-click interface with OpenAI generative AI. Upload your contextual coursework (book chapters, etc), and chat with your own customized tutor!
+* **[Hosted self- and in-class oral exam app]()**: Interested in using oral exams for assessment of student knowledge? Students - want to prepare for oral exams? Visit our point-and-click interface with OpenAI generative AI using your own coursework. You can upload questions provided by the instructor or ask the generative AI to assist in creating questions for you.
+
+### **Google Colab Notebooks**
+Want to evaluate students or want to customize your approach for interacting with generative AI?
+
+* **[Instructor Grading Notebook](https://github.com/vanderbilt-data-science/lo-achievement/blob/main/instructor_intr_notebook.ipynb)**: Upload a zip file of the JSON output of your students' exploration and assessment, and get insight on student strengths and weaknesses on the topic as well as structured feedback for all students.
+* **[Instructor Document Store Creation](https://github.com/vanderbilt-data-science/lo-achievement/blob/main/instructor_vector_store_creator.ipynb)**: Store all of your classroom content (pdfs, Youtube videos, website links) in a hosted location for easy accessibility for students working with generative AI platforms.
+* **[Prompting with Inline Context](https://github.com/vanderbilt-data-science/lo-achievement/blob/main/prompt_with_context.ipynb)**: A customizable, programmatic way to interface with generative AI using Google Colab through direct copy/paste of text content
+* **[Prompting with Vector Stores](https://github.com/vanderbilt-data-science/lo-achievement/blob/main/prompt_with_vector_store.ipynb)**: A customizable, programming way for self-study with generative AI using Google Colab through the creation of vector stores (better for larger corpuses of text)
+
+**Use the repo!**
+You can also directly clone/use the repo itself or use it as a package for development - this is great for experienced programmers or even those who would like to learn more about development with generative AI. The repo provides:
+- Generative AI and LangChain integration to process sources and create assessments and answer keys
+- Runs on Google Colab, with no additional installations needed
+You can also develop locally as desired and we encourage PR contributions from the community!
+
+## Getting Started
+
+There are a variety of ways to use CLAS:
+* **Navigate to the Wiki to explore prompts.** You can copy/paste/amend these in the interfaces provided by OpenAI, Google, Anthropic, etc.
+* **Use Google Colab to interact with notebooks.** Click on the notebook you'd like to enter in the files list above. You will see a blue Open In Colab link in the page that opens. Click this button to start your session in Google Colab, making sure that you're logged in with your Google Account.** It will take a few minutes to spin up and automatically install the required packages.
+* **Study through our hosted app.** Navigate to [CLAS on Huggingface](https://huggingface.co/spaces/vanderbilt-dsi/selfstudy_learningobjectives_demo). Follow the instructions to use the platform for self study.
+
+## Contributing
+
+To contribute to the project, please fork the repository and submit a pull request. Our community is supportive, and we provide training and classes if you're new to any of the frameworks used in the project. Everyone is welcome to contribute, as we believe participating in data science and AI projects is an excellent way to learn.
+
+## Community Guidelines
+
+We aim to create a welcoming and inclusive community where everyone can feel comfortable and valued, regardless of skill level, background, ability, or identity. To ensure a positive atmosphere, please adhere to our code of conduct and community guidelines.
+
+## Meetings
+
+- Sprint Planning & Retrospective: Mondays and Fridays at 10:30 am
+- Demos: Fridays at 3 pm
+
+## Additional Resources
+
+- LangChain documentation
+- Introduction to transformers and generative AI on our [YouTube channel](https://www.youtube.com/channel/UC8C2_3L5gR9qLmL7rmb2BdQ)
+- AI Summer and AI Winter sessions (free and open to all)
+
+## Reporting Issues
+
+If you encounter a bug, please submit an issue and label it with "Bug." To escalate the issue, email [[email protected]](mailto:[email protected]).
+
+## Contact Information
+
+- Organization: Data Science Institute at Vanderbilt University
+- Program: Data Science for Social Good
+- Main Email: [[email protected]](mailto:[email protected])
+- Principal Investigators (PIs)
+  - Jesse Spencer-Smith, Ph.D., Chief Data Scientist, Data Science Institute, Vanderbilt University
+  - Jesse Blocher, Ph.D., Director of Graduate Studies, Data Science Institute, Vanderbilt University
+  - Dr. Yaa Kumah-Crystal, Ph.D., Pediatric Endocrinologist and Professor, Vanderbilt University Medical Center
+  - Charreau Bell, Ph.D., Senior Data Scientist, Data Science Institute, Vanderbilt University
+- Staff Lead: [[email protected]](mailto:[email protected])  
+- Code Developers:
+  - Katrina Rbeiz, Ph.D. Student, Psychology, Vanderbilt University
+  - Minwoo Sohn, Graduate Student, Data Science, Vanderbilt University
+  - Ricky Sun, Graduate Student, Data Science, Vanderbilt University
+  - Eleanor Beers, Graduate Student, Data Science, Vanderbilt University
+  - Kevin Chen, Undergraduate, Computer Science, Vanderbilt University
+  - Adam Levav, Undergraduate, University of Maryland
+  - Varun Koduvayur, Undergraduate

lo-achievement/UI_design_oral_exam_baseline_functionality.ipynb

@@ -0,0 +1,1454 @@
+{
+  "nbformat": 4,
+  "nbformat_minor": 0,
+  "metadata": {
+    "colab": {
+      "provenance": [],
+      "gpuType": "V100",
+      "include_colab_link": true
+    },
+    "kernelspec": {
+      "name": "python3",
+      "display_name": "Python 3"
+    },
+    "language_info": {
+      "name": "python"
+    }
+  },
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "view-in-github",
+        "colab_type": "text"
+      },
+      "source": [
+        "<a href=\"https://colab.research.google.com/github/vanderbilt-data-science/lo-achievement/blob/124-implement-baseline-functionality-for-oral-exam-module/UI_design_oral_exam_baseline_functionality.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "# Project IO Achievement - UI Design (Oral Exam)"
+      ],
+      "metadata": {
+        "id": "PIbogPXyM0wr"
+      }
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## Problem Definition\n",
+        "\n",
+        "The v1 functionality for the Oral Exam module requires the following:\n",
+        "\n",
+        "1. Upload or generation of questions: either the user should upload a set of questions or we should allow the model to generate the questions. The user should pick or it should be inherent if there is no upload of questions. Note that we must also allow for context to be uploaded (vector store, vector store link, specific documents)\n",
+        "2. The model should prompt the user with a question and pause.\n",
+        "The user should respond by audio.\n",
+        "3. This should continue on until some final point where the exam is over.\n",
+        "\n",
+        "Then:\n",
+        "\n",
+        "1. We should use Whisper to do the transcription, and\n",
+        "2. Send the transcription, questions, and context for GPT4 for evaluation\n",
+        "Return the evaluation.\n",
+        "3. This will primarily be work on a user interface."
+      ],
+      "metadata": {
+        "id": "x_Vp8SiKM4p1"
+      }
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## Libraries\n",
+        "\n",
+        "This section will install and import some important libraries such as Langchain, openai, Gradio, and so on"
+      ],
+      "metadata": {
+        "id": "o_60X8H3NEne"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# install libraries here\n",
+        "# -q flag for \"quiet\" install\n",
+        "%%capture\n",
+        "!pip install -q langchain\n",
+        "!pip install -q openai\n",
+        "!pip install -q gradio\n",
+        "!pip install -q transformers\n",
+        "!pip install -q datasets\n",
+        "!pip install -q huggingsound\n",
+        "!pip install -q torchaudio\n",
+        "!pip install -q git+https://github.com/openai/whisper.git\n",
+        "!pip install -q docx\n",
+        "!pip install -q PyPDF2\n",
+        "!pip install python-docx"
+      ],
+      "metadata": {
+        "id": "pxcqXgg2aAN7"
+      },
+      "execution_count": 1,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 2,
+      "metadata": {
+        "id": "pEjM1tLsMZBq"
+      },
+      "outputs": [],
+      "source": [
+        "# import libraries here\n",
+        "from langchain.llms import OpenAI\n",
+        "from langchain.prompts import PromptTemplate\n",
+        "from langchain.document_loaders import TextLoader\n",
+        "from langchain.indexes import VectorstoreIndexCreator\n",
+        "from langchain import ConversationChain, LLMChain, PromptTemplate\n",
+        "from langchain.chat_models import ChatOpenAI\n",
+        "from langchain.memory import ConversationBufferWindowMemory\n",
+        "from langchain.prompts import ChatPromptTemplate\n",
+        "from langchain.text_splitter import CharacterTextSplitter\n",
+        "from langchain.embeddings import OpenAIEmbeddings\n",
+        "from langchain.schema import SystemMessage, HumanMessage, AIMessage\n",
+        "import openai\n",
+        "import os\n",
+        "from getpass import getpass\n",
+        "from IPython.display import display, Javascript, HTML\n",
+        "from google.colab.output import eval_js\n",
+        "from base64 import b64decode\n",
+        "import ipywidgets as widgets\n",
+        "from IPython.display import clear_output\n",
+        "import time\n",
+        "import requests\n",
+        "from transformers import WhisperProcessor, WhisperForConditionalGeneration\n",
+        "from datasets import load_dataset\n",
+        "# from torchaudio.transforms import Resample\n",
+        "import whisper\n",
+        "from huggingsound import SpeechRecognitionModel\n",
+        "import numpy as np\n",
+        "import torch\n",
+        "import librosa\n",
+        "from datasets import load_dataset\n",
+        "from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor\n",
+        "from jiwer import wer\n",
+        "import pandas as pd\n",
+        "from IPython.display import display, HTML\n",
+        "import gradio as gr\n",
+        "from transformers import pipeline\n",
+        "from docx import Document\n",
+        "import PyPDF2\n",
+        "from pydub import AudioSegment\n",
+        "import tempfile\n",
+        "import os\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## API Keys\n",
+        "\n",
+        "Use these cells to load the API keys required for this notebook. The below code cell uses the `getpass` library."
+      ],
+      "metadata": {
+        "id": "03KLZGI_a5W5"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "openai_api_key = getpass()\n",
+        "os.environ[\"OPENAI_API_KEY\"] = openai_api_key\n",
+        "openai.api_key = openai_api_key"
+      ],
+      "metadata": {
+        "id": "5smcWj4DbFgy",
+        "outputId": "9a73707b-1a6a-4253-b7d8-181a82b1040f",
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        }
+      },
+      "execution_count": 3,
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "··········\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## Prompt Design\n",
+        "\n",
+        "To be added"
+      ],
+      "metadata": {
+        "id": "pMo9x8u4AEV1"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "chat = ChatOpenAI(temperature=0.0, model_name='gpt-3.5-turbo-16k')\n",
+        "chat"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "UgnCZRMhADvo",
+        "outputId": "1bd6b84d-3ea8-49ba-8156-701f4155d69c"
+      },
+      "execution_count": 4,
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "ChatOpenAI(cache=None, verbose=False, callbacks=None, callback_manager=None, tags=None, metadata=None, client=<class 'openai.api_resources.chat_completion.ChatCompletion'>, model_name='gpt-3.5-turbo-16k', temperature=0.0, model_kwargs={}, openai_api_key='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', openai_api_base='', openai_organization='', openai_proxy='', request_timeout=None, max_retries=6, streaming=False, n=1, max_tokens=None, tiktoken_model_name=None)"
+            ]
+          },
+          "metadata": {},
+          "execution_count": 4
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# This is what I used to test the function 'generate_questions'\n",
+        "template_string2 = \"\"\"\n",
+        "You are teacher, and you will be given a {context} that is related to the presentation topic.\n",
+        "\n",
+        "Please generate a questions based on the context above and transcript that student created . \\\n",
+        "\n",
+        "The audio file generated by student is shown below: {transcribed_text}. \\\n",
+        "\"\"\""
+      ],
+      "metadata": {
+        "id": "WmysQZAhKBli"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "prompt_template1 = ChatPromptTemplate.from_template(template_string2)"
+      ],
+      "metadata": {
+        "id": "oij6W5rwAaGb"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# prompt_template.messages[0].prompt\n",
+        "prompt_template1.messages[0].prompt.input_variables"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "C1YRmL46AaJA",
+        "outputId": "e850524a-3831-4113-c796-5a0ec8584569"
+      },
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "['context', 'transcribed_text']"
+            ]
+          },
+          "metadata": {},
+          "execution_count": 25
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# This template is used for testing the function 'ai_evaluate'\n",
+        "# Detailed evaluation metrics are to be added\n",
+        "template_string3 = \"\"\"\n",
+        "You are teacher, and you will be given a context that is related to the presentation topic. \\\n",
+        "Now, given {context}, evaluate the answer based on the accuracy\n",
+        "\n",
+        "The main answer generated by student is shown below: {transcribed_text}. \\\n",
+        "The questions are shown below: {questions}. \\\n",
+        "The questions answered by student is shown below: {transcribed_qa}. \\\n",
+        "\"\"\"\n",
+        "prompt_template2 = ChatPromptTemplate.from_template(template_string3)\n",
+        "prompt_template2.messages[0].prompt.input_variables\n"
+      ],
+      "metadata": {
+        "id": "141Cxa2MT-l7",
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "outputId": "0374d45d-a7f6-41e7-aed0-44c61681de21"
+      },
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "['context', 'questions', 'transcribed_qa', 'transcribed_text']"
+            ]
+          },
+          "metadata": {},
+          "execution_count": 7
+        }
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## Integrate Prompts from LO project"
+      ],
+      "metadata": {
+        "id": "MJCHl1T2TPWC"
+      }
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "### Creating a Chain for Short Answer Generation"
+      ],
+      "metadata": {
+        "id": "IPTyUOl-WdiL"
+      }
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "In this example, the context would include the poem \"The Road Not Taken\" by Robert Frost"
+      ],
+      "metadata": {
+        "id": "203qBjZvmFK1"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# This is what I used to test the function 'generate_questions_v2'\n",
+        "template_string = \"\"\"\n",
+        "You are a world-class tutor helping students to perform better on oral and written exams though interactive experiences.\"\n",
+        "\n",
+        "The following text should be used as the basis for the instructions which follow: {context} \\\n",
+        "\n",
+        "The following is the guideline for generating the questiion: {pre_prompt}\n",
+        "\"\"\""
+      ],
+      "metadata": {
+        "id": "w1AjHwIoVnvw"
+      },
+      "execution_count": 5,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "prompt_template = ChatPromptTemplate.from_template(template_string)\n",
+        "prompt_template.messages[0].prompt.input_variables"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "39-lm5I-Wlep",
+        "outputId": "44b39930-5258-484b-8c7d-c36ff4b5dc1a"
+      },
+      "execution_count": 6,
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "['context', 'pre_prompt']"
+            ]
+          },
+          "metadata": {},
+          "execution_count": 6
+        }
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "### Creating a Chain for AI Evaluation"
+      ],
+      "metadata": {
+        "id": "-Pfxkcdxh9nZ"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "template_evaluation = \"\"\"\n",
+        "Given the follwing {context} and the {transcript}, evaluate whether or not the student answered correctly on the {question}.\n",
+        "\"\"\""
+      ],
+      "metadata": {
+        "id": "u6pH1x-gWnFF"
+      },
+      "execution_count": 7,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# @title\n",
+        "prompt_template2 = ChatPromptTemplate.from_template(template_evaluation)\n",
+        "prompt_template2.messages[0].prompt.input_variables"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "YPO_IE5ThC6W",
+        "outputId": "5361929c-cf8c-483d-901a-ed14a0db89fa"
+      },
+      "execution_count": 8,
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "['context', 'question', 'transcript']"
+            ]
+          },
+          "metadata": {},
+          "execution_count": 8
+        }
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## UI Design\n",
+        "\n",
+        "https://colab.research.google.com/github/petewarden/openai-whisper-webapp/blob/main/OpenAI_Whisper_ASR_Demo.ipynb"
+      ],
+      "metadata": {
+        "id": "M6IzVTjz5cex"
+      }
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "### Functions"
+      ],
+      "metadata": {
+        "id": "l4o8R5eUE1n8"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "def embed_key(openai_api_key):\n",
+        "  os.environ[\"OPENAI_API_KEY\"] = openai_api_key\n",
+        "\n",
+        "def transcribe(audio_file_path):\n",
+        "    with open(audio_file_path, \"rb\") as audio_file:\n",
+        "        # Call OpenAI's Whisper model for transcription\n",
+        "        transcript = openai.Audio.transcribe(\"whisper-1\", audio_file)\n",
+        "        transcribed_text = transcript[\"text\"]\n",
+        "        return transcribed_text\n",
+        "\n",
+        "def translate(text):\n",
+        "    # Create a prompt template (This will be changed later to fit the actual task)\n",
+        "    # Here translation will be a filler task of GPT\n",
+        "    test_input1 = prompt_template.format_messages(\n",
+        "              expertise='Language Translation',\n",
+        "              language='Japanese',\n",
+        "              style='romantic',\n",
+        "              transcribed_text=text)\n",
+        "\n",
+        "    response = chat.predict_messages(test_input1)\n",
+        "    return response.content\n",
+        "\n",
+        "def process_file(files):\n",
+        "  for file in files:\n",
+        "    try:\n",
+        "        extension = file.name.split('.')[-1].lower()\n",
+        "        if extension == 'docx':\n",
+        "            doc = Document(file.name)\n",
+        "            full_text = []\n",
+        "            for paragraph in doc.paragraphs:\n",
+        "                full_text.append(paragraph.text)\n",
+        "            return '\\n'.join(full_text)\n",
+        "\n",
+        "        elif extension == 'pdf':\n",
+        "            pdf_file = open(file.name, 'rb')\n",
+        "            reader = PyPDF2.PdfReader(pdf_file)\n",
+        "            num_pages = len(reader.pages)\n",
+        "            full_text = []\n",
+        "            for page in range(num_pages):\n",
+        "                page_obj = reader.pages[page]\n",
+        "                full_text.append(page_obj.extract_text())\n",
+        "            pdf_file.close()\n",
+        "            return '\\n'.join(full_text)\n",
+        "\n",
+        "        elif extension == 'txt':\n",
+        "            with open(file.name, 'r') as txt_file:\n",
+        "                full_text = txt_file.read()\n",
+        "            return full_text\n",
+        "\n",
+        "        else:\n",
+        "            return \"Unsupported file type\"\n",
+        "    except FileNotFoundError:\n",
+        "        return \"File not found\"\n",
+        "    except PermissionError:\n",
+        "        return \"Permission denied\"\n",
+        "\n",
+        "def generate_questions(context, transcript):\n",
+        "    text = process_file(context)\n",
+        "    test_input1 = prompt_template1.format_messages(\n",
+        "                      context = text,\n",
+        "                      transcribed_text = transcript)\n",
+        "\n",
+        "    response = chat(test_input1)\n",
+        "    return response.content\n",
+        "\n",
+        "def generate_questions_v2(text, prompt):\n",
+        "    #text = process_file(file)\n",
+        "    test_input1 = prompt_template.format_messages(\n",
+        "                      context = text,\n",
+        "                      pre_prompt = prompt)\n",
+        "\n",
+        "    response = chat(test_input1)\n",
+        "    return response\n",
+        "\n",
+        "# def ai_evaluate(context, audio_main, audio_qa, questions):\n",
+        "#     test_input1 = prompt_template2.format_messages(\n",
+        "#                       context = context,\n",
+        "#                       transcribed_text = audio_main,\n",
+        "#                       transcribed_qa = audio_qa,\n",
+        "#                       questions = questions)\n",
+        "\n",
+        "#     response = chat(test_input1)\n",
+        "#     return response.content\n",
+        "\n",
+        "def ai_evaluate_v2(text, audio_main, questions):\n",
+        "    #audio = transcribe(audio_main)\n",
+        "    test_input1 = prompt_template2.format_messages(\n",
+        "                      context = text,\n",
+        "                      transcript = audio_main,\n",
+        "                      question = questions\n",
+        "                      )\n",
+        "\n",
+        "    response = chat(test_input1)\n",
+        "    return response.content\n",
+        "\n",
+        "def upload_file(files):\n",
+        "    file_paths = [file.name for file in files]\n",
+        "    return file_paths"
+      ],
+      "metadata": {
+        "id": "ABN0X9xQHeii"
+      },
+      "execution_count": 12,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "### Test process_file"
+      ],
+      "metadata": {
+        "id": "a3WUL_hFyMkr"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from google.colab import files\n",
+        "def upload_syllabi():\n",
+        "  uploaded = files.upload()\n",
+        "  for name, data in uploaded.items():\n",
+        "    with open(name, 'wb') as f:\n",
+        "      f.write(data)\n",
+        "    print('saved file', name)\n",
+        "upload_syllabi()"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 90
+        },
+        "id": "nih4FXX0Pl9U",
+        "outputId": "ce48c70a-d52c-4267-f3fc-8b22404448d7"
+      },
+      "execution_count": 13,
+      "outputs": [
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "<IPython.core.display.HTML object>"
+            ],
+            "text/html": [
+              "\n",
+              "     <input type=\"file\" id=\"files-c72e8e8a-ac6f-48ab-9fb4-a84b1483268a\" name=\"files[]\" multiple disabled\n",
+              "        style=\"border:none\" />\n",
+              "     <output id=\"result-c72e8e8a-ac6f-48ab-9fb4-a84b1483268a\">\n",
+              "      Upload widget is only available when the cell has been executed in the\n",
+              "      current browser session. Please rerun this cell to enable.\n",
+              "      </output>\n",
+              "      <script>// Copyright 2017 Google LLC\n",
+              "//\n",
+              "// Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+              "// you may not use this file except in compliance with the License.\n",
+              "// You may obtain a copy of the License at\n",
+              "//\n",
+              "//      http://www.apache.org/licenses/LICENSE-2.0\n",
+              "//\n",
+              "// Unless required by applicable law or agreed to in writing, software\n",
+              "// distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+              "// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+              "// See the License for the specific language governing permissions and\n",
+              "// limitations under the License.\n",
+              "\n",
+              "/**\n",
+              " * @fileoverview Helpers for google.colab Python module.\n",
+              " */\n",
+              "(function(scope) {\n",
+              "function span(text, styleAttributes = {}) {\n",
+              "  const element = document.createElement('span');\n",
+              "  element.textContent = text;\n",
+              "  for (const key of Object.keys(styleAttributes)) {\n",
+              "    element.style[key] = styleAttributes[key];\n",
+              "  }\n",
+              "  return element;\n",
+              "}\n",
+              "\n",
+              "// Max number of bytes which will be uploaded at a time.\n",
+              "const MAX_PAYLOAD_SIZE = 100 * 1024;\n",
+              "\n",
+              "function _uploadFiles(inputId, outputId) {\n",
+              "  const steps = uploadFilesStep(inputId, outputId);\n",
+              "  const outputElement = document.getElementById(outputId);\n",
+              "  // Cache steps on the outputElement to make it available for the next call\n",
+              "  // to uploadFilesContinue from Python.\n",
+              "  outputElement.steps = steps;\n",
+              "\n",
+              "  return _uploadFilesContinue(outputId);\n",
+              "}\n",
+              "\n",
+              "// This is roughly an async generator (not supported in the browser yet),\n",
+              "// where there are multiple asynchronous steps and the Python side is going\n",
+              "// to poll for completion of each step.\n",
+              "// This uses a Promise to block the python side on completion of each step,\n",
+              "// then passes the result of the previous step as the input to the next step.\n",
+              "function _uploadFilesContinue(outputId) {\n",
+              "  const outputElement = document.getElementById(outputId);\n",
+              "  const steps = outputElement.steps;\n",
+              "\n",
+              "  const next = steps.next(outputElement.lastPromiseValue);\n",
+              "  return Promise.resolve(next.value.promise).then((value) => {\n",
+              "    // Cache the last promise value to make it available to the next\n",
+              "    // step of the generator.\n",
+              "    outputElement.lastPromiseValue = value;\n",
+              "    return next.value.response;\n",
+              "  });\n",
+              "}\n",
+              "\n",
+              "/**\n",
+              " * Generator function which is called between each async step of the upload\n",
+              " * process.\n",
+              " * @param {string} inputId Element ID of the input file picker element.\n",
+              " * @param {string} outputId Element ID of the output display.\n",
+              " * @return {!Iterable<!Object>} Iterable of next steps.\n",
+              " */\n",
+              "function* uploadFilesStep(inputId, outputId) {\n",
+              "  const inputElement = document.getElementById(inputId);\n",
+              "  inputElement.disabled = false;\n",
+              "\n",
+              "  const outputElement = document.getElementById(outputId);\n",
+              "  outputElement.innerHTML = '';\n",
+              "\n",
+              "  const pickedPromise = new Promise((resolve) => {\n",
+              "    inputElement.addEventListener('change', (e) => {\n",
+              "      resolve(e.target.files);\n",
+              "    });\n",
+              "  });\n",
+              "\n",
+              "  const cancel = document.createElement('button');\n",
+              "  inputElement.parentElement.appendChild(cancel);\n",
+              "  cancel.textContent = 'Cancel upload';\n",
+              "  const cancelPromise = new Promise((resolve) => {\n",
+              "    cancel.onclick = () => {\n",
+              "      resolve(null);\n",
+              "    };\n",
+              "  });\n",
+              "\n",
+              "  // Wait for the user to pick the files.\n",
+              "  const files = yield {\n",
+              "    promise: Promise.race([pickedPromise, cancelPromise]),\n",
+              "    response: {\n",
+              "      action: 'starting',\n",
+              "    }\n",
+              "  };\n",
+              "\n",
+              "  cancel.remove();\n",
+              "\n",
+              "  // Disable the input element since further picks are not allowed.\n",
+              "  inputElement.disabled = true;\n",
+              "\n",
+              "  if (!files) {\n",
+              "    return {\n",
+              "      response: {\n",
+              "        action: 'complete',\n",
+              "      }\n",
+              "    };\n",
+              "  }\n",
+              "\n",
+              "  for (const file of files) {\n",
+              "    const li = document.createElement('li');\n",
+              "    li.append(span(file.name, {fontWeight: 'bold'}));\n",
+              "    li.append(span(\n",
+              "        `(${file.type || 'n/a'}) - ${file.size} bytes, ` +\n",
+              "        `last modified: ${\n",
+              "            file.lastModifiedDate ? file.lastModifiedDate.toLocaleDateString() :\n",
+              "                                    'n/a'} - `));\n",
+              "    const percent = span('0% done');\n",
+              "    li.appendChild(percent);\n",
+              "\n",
+              "    outputElement.appendChild(li);\n",
+              "\n",
+              "    const fileDataPromise = new Promise((resolve) => {\n",
+              "      const reader = new FileReader();\n",
+              "      reader.onload = (e) => {\n",
+              "        resolve(e.target.result);\n",
+              "      };\n",
+              "      reader.readAsArrayBuffer(file);\n",
+              "    });\n",
+              "    // Wait for the data to be ready.\n",
+              "    let fileData = yield {\n",
+              "      promise: fileDataPromise,\n",
+              "      response: {\n",
+              "        action: 'continue',\n",
+              "      }\n",
+              "    };\n",
+              "\n",
+              "    // Use a chunked sending to avoid message size limits. See b/62115660.\n",
+              "    let position = 0;\n",
+              "    do {\n",
+              "      const length = Math.min(fileData.byteLength - position, MAX_PAYLOAD_SIZE);\n",
+              "      const chunk = new Uint8Array(fileData, position, length);\n",
+              "      position += length;\n",
+              "\n",
+              "      const base64 = btoa(String.fromCharCode.apply(null, chunk));\n",
+              "      yield {\n",
+              "        response: {\n",
+              "          action: 'append',\n",
+              "          file: file.name,\n",
+              "          data: base64,\n",
+              "        },\n",
+              "      };\n",
+              "\n",
+              "      let percentDone = fileData.byteLength === 0 ?\n",
+              "          100 :\n",
+              "          Math.round((position / fileData.byteLength) * 100);\n",
+              "      percent.textContent = `${percentDone}% done`;\n",
+              "\n",
+              "    } while (position < fileData.byteLength);\n",
+              "  }\n",
+              "\n",
+              "  // All done.\n",
+              "  yield {\n",
+              "    response: {\n",
+              "      action: 'complete',\n",
+              "    }\n",
+              "  };\n",
+              "}\n",
+              "\n",
+              "scope.google = scope.google || {};\n",
+              "scope.google.colab = scope.google.colab || {};\n",
+              "scope.google.colab._files = {\n",
+              "  _uploadFiles,\n",
+              "  _uploadFilesContinue,\n",
+              "};\n",
+              "})(self);\n",
+              "</script> "
+            ]
+          },
+          "metadata": {}
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Saving instructor_note_2.docx to instructor_note_2.docx\n",
+            "saved file instructor_note_2.docx\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# Might need some way to make pdf file to load more readable\n",
+        "# process_file('/content/instrutor_note.docx')\n",
+        "# process_file('/content/Big Data & Economics.pdf')\n",
+        "# process_file('/content/Big Data & Economics.pdf')"
+      ],
+      "metadata": {
+        "id": "LJX1AKTMyVm8"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "### Gradio Interface V1"
+      ],
+      "metadata": {
+        "id": "c4s5o8baE6wN"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# @title\n",
+        "# with gr.Blocks() as demo:\n",
+        "#   gr.Markdown(\"# Oral Exam App\")\n",
+        "#   with gr.Box():\n",
+        "#     gr.HTML(\"\"\"Embed your OpenAI API key below; if you haven't created one already, visit\n",
+        "#       platform.openai.com/account/api-keys\n",
+        "#     to sign up for an account and get your personal API key\"\"\",\n",
+        "#             elem_classes=\"textbox_label\")\n",
+        "#     input = gr.Textbox(show_label=False, type=\"password\", container=False,\n",
+        "#                       placeholder=\"●●●●●●●●●●●●●●●●●\")\n",
+        "#     input.change(fn=embed_key, inputs=input, outputs=None)\n",
+        "\n",
+        "#   with gr.Blocks():\n",
+        "#       gr.Markdown(\"## Upload your audio file or start recording\")\n",
+        "\n",
+        "#       with gr.Row():\n",
+        "\n",
+        "\n",
+        "#         with gr.Column():\n",
+        "#           file_input = gr.Files(label=\"Load a mp3 file\",\n",
+        "#                               file_types=['.mp3'], type=\"file\",\n",
+        "#                               elem_classes=\"short-height\")\n",
+        "#           record_inputs = gr.Audio(source=\"microphone\", type=\"filepath\")\n",
+        "\n",
+        "#         with gr.Column():\n",
+        "#           outputs_transcribe=gr.Textbox(label=\"Transcription\")\n",
+        "\n",
+        "#           with gr.Row():\n",
+        "#             btn1 = gr.Button(value=\"Transcribe recorded audio\")\n",
+        "#             btn1.click(transcribe, inputs=record_inputs, outputs=outputs_transcribe)\n",
+        "#             btn2 = gr.Button(value=\"Transcribe uploaded audio\")\n",
+        "#             btn2.click(transcribe, inputs=file_input, outputs=outputs_transcribe)\n",
+        "\n",
+        "#           outputs_translate=gr.Textbox(label=\"Translation\")\n",
+        "#           btn3 = gr.Button(value=\"Translate\")\n",
+        "#           btn3.click(translate, inputs=outputs_transcribe, outputs=outputs_translate)\n",
+        "\n",
+        "#   demo.launch()\n"
+      ],
+      "metadata": {
+        "id": "ZkfJXCGDFhdw",
+        "cellView": "form"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "### baseline functionality V1"
+      ],
+      "metadata": {
+        "id": "AnkuosJ7Vw4z"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# @title\n",
+        "with gr.Blocks() as demo:\n",
+        "  gr.Markdown(\"# Oral Exam App\")\n",
+        "  gr.Markdown(\"## OpenAI API key\")\n",
+        "  with gr.Box():\n",
+        "    gr.HTML(\"\"\"Embed your OpenAI API key below; if you haven't created one already, visit\n",
+        "      platform.openai.com/account/api-keys\n",
+        "    to sign up for an account and get your personal API key\"\"\",\n",
+        "            elem_classes=\"textbox_label\")\n",
+        "    input = gr.Textbox(show_label=False, type=\"password\", container=False,\n",
+        "                      placeholder=\"●●●●●●●●●●●●●●●●●\")\n",
+        "    input.change(fn=embed_key, inputs=input, outputs=None)\n",
+        "\n",
+        "  with gr.Blocks():\n",
+        "  #########################\n",
+        "  #########Context#########\n",
+        "  #########################\n",
+        "      with gr.Accordion(\"Context section\"):\n",
+        "      ### Should also allow vector stores\n",
+        "          gr.Markdown(\"## Please upload the context document(s) for Oral exam\")\n",
+        "          context_input = gr.File(label=\"Click to upload context file\",\n",
+        "                                  file_count=\"multiple\",\n",
+        "                                  file_types=[\".txt\", \".docx\", \".pdf\"])\n",
+        "          outputs_context=gr.Textbox(label=\"Context\")\n",
+        "          context_input.change(fn=process_file, inputs=context_input, outputs=outputs_context)\n",
+        "          # upload_button = gr.Button(value=\"Show context\")\n",
+        "          # upload_button.click(process_file, context_input, outputs_context)\n",
+        "\n",
+        "  #########################\n",
+        "  #######Main Audio########\n",
+        "  #########################\n",
+        "      with gr.Accordion(\"Main audio section\"):\n",
+        "        gr.Markdown(\"## Upload your audio file or start recording\")\n",
+        "        with gr.Column():\n",
+        "          ## uploading files seem not working (don't know why)\n",
+        "          with gr.Row():\n",
+        "              file_input = gr.Audio(label=\"Upload Audio\", source=\"upload\", type=\"filepath\")\n",
+        "              record_inputs = gr.Audio(label=\"Record Audio\", source=\"microphone\", type=\"filepath\")\n",
+        "\n",
+        "          gr.Markdown(\"## Transcribe the audio uploaded or recorded\")\n",
+        "          outputs_transcribe=gr.Textbox(label=\"Transcription\")\n",
+        "\n",
+        "          file_input.change(fn=transcribe, inputs=file_input, outputs=outputs_transcribe)\n",
+        "          record_inputs.change(fn=transcribe, inputs=record_inputs, outputs=outputs_transcribe)\n",
+        "\n",
+        "  #########################\n",
+        "  ###Question Generation###\n",
+        "  #########################\n",
+        "      with gr.Accordion(\"Question section\"):\n",
+        "          gr.Markdown(\"## Questions\")\n",
+        "          with gr.Row():\n",
+        "            with gr.Column():\n",
+        "              outputs_qa=gr.Textbox(label=\"Generate questions\")\n",
+        "              btn3 = gr.Button(value=\"Generate questions\")\n",
+        "              btn3.click(generate_questions, inputs=[context_input, outputs_transcribe], outputs=outputs_qa)\n",
+        "\n",
+        "  ######################### Need additional work to include these questions when click button   #########################\n",
+        "            with gr.Column():\n",
+        "              submit_question=gr.Textbox(label=\"Use existing questions\")\n",
+        "              btn4 = gr.Button(value=\"Use these questions\")\n",
+        "              # btn4.click(use_this_question, inputs=outputs_transcribe, outputs=None)\n",
+        "\n",
+        "  #########################\n",
+        "  #########Audio QA########\n",
+        "  #########################\n",
+        "      with gr.Accordion(\"Audio QA section\"):\n",
+        "          gr.Markdown(\"## Question answering\")\n",
+        "          ##### This may be iterative\n",
+        "          with gr.Row():\n",
+        "            file_input2 = gr.Audio(label=\"Upload Audio\", source=\"upload\", type=\"filepath\")\n",
+        "            record_inputs2 = gr.Audio(label=\"Record Audio\", source=\"microphone\", type=\"filepath\")\n",
+        "\n",
+        "          gr.Markdown(\"## Transcribe the audio uploaded or recorded\")\n",
+        "          outputs_transcribe2=gr.Textbox(label=\"Transcription\")\n",
+        "          file_input2.change(fn=transcribe, inputs=file_input2, outputs=outputs_transcribe2)\n",
+        "          record_inputs2.change(fn=transcribe, inputs=record_inputs2, outputs=outputs_transcribe2)\n",
+        "\n",
+        "  #########################\n",
+        "  #######Evaluation########\n",
+        "  #########################\n",
+        "      with gr.Accordion(\"Evaluation section\"):\n",
+        "          gr.Markdown(\"## Evaluation\")\n",
+        "          with gr.Tab(\"General evalution\"):\n",
+        "            evalution=gr.Textbox(label=\"AI Evaluation\")\n",
+        "            btn5 = gr.Button(value=\"Evaluate\")\n",
+        "            btn5.click(ai_evaluate, inputs=[context_input, record_inputs,record_inputs2, outputs_qa], outputs=evalution)\n",
+        "          with gr.Tab(\"Quantitative evalution\"):\n",
+        "            table_output = gr.Dataframe(label = \"Some kind of evaluation metrics?\")\n",
+        "            btn6 = gr.Button(value=\"Evaluate\")\n",
+        "            btn6.click(ai_evaluate, inputs=[context_input, record_inputs,record_inputs2, outputs_qa], outputs=table_output)\n",
+        "\n",
+        "  demo.launch()\n",
+        "  # demo.launch(share=True)\n",
+        "  # demo.launch(debug=True)"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 616
+        },
+        "id": "EAPljDMYVy3u",
+        "outputId": "1f347376-14e8-48ea-e531-295a4fefd6cd",
+        "cellView": "form"
+      },
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Colab notebook detected. To show errors in colab notebook, set debug=True in launch()\n",
+            "Note: opening Chrome Inspector may crash demo inside Colab notebooks.\n",
+            "\n",
+            "To create a public link, set `share=True` in `launch()`.\n"
+          ]
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "<IPython.core.display.Javascript object>"
+            ],
+            "application/javascript": [
+              "(async (port, path, width, height, cache, element) => {\n",
+              "                        if (!google.colab.kernel.accessAllowed && !cache) {\n",
+              "                            return;\n",
+              "                        }\n",
+              "                        element.appendChild(document.createTextNode(''));\n",
+              "                        const url = await google.colab.kernel.proxyPort(port, {cache});\n",
+              "\n",
+              "                        const external_link = document.createElement('div');\n",
+              "                        external_link.innerHTML = `\n",
+              "                            <div style=\"font-family: monospace; margin-bottom: 0.5rem\">\n",
+              "                                Running on <a href=${new URL(path, url).toString()} target=\"_blank\">\n",
+              "                                    https://localhost:${port}${path}\n",
+              "                                </a>\n",
+              "                            </div>\n",
+              "                        `;\n",
+              "                        element.appendChild(external_link);\n",
+              "\n",
+              "                        const iframe = document.createElement('iframe');\n",
+              "                        iframe.src = new URL(path, url).toString();\n",
+              "                        iframe.height = height;\n",
+              "                        iframe.allow = \"autoplay; camera; microphone; clipboard-read; clipboard-write;\"\n",
+              "                        iframe.width = width;\n",
+              "                        iframe.style.border = 0;\n",
+              "                        element.appendChild(iframe);\n",
+              "                    })(7862, \"/\", \"100%\", 500, false, window.element)"
+            ]
+          },
+          "metadata": {}
+        }
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "### Baseline Functionality V2"
+      ],
+      "metadata": {
+        "id": "YAKl-5P4dHEF"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# @title\n",
+        "with gr.Blocks() as demo:\n",
+        "  gr.Markdown(\"# Oral Exam App\")\n",
+        "  gr.Markdown(\"## OpenAI API key\")\n",
+        "  with gr.Box():\n",
+        "    gr.HTML(\"\"\"Embed your OpenAI API key below; if you haven't created one already, visit\n",
+        "      platform.openai.com/account/api-keys\n",
+        "    to sign up for an account and get your personal API key\"\"\",\n",
+        "            elem_classes=\"textbox_label\")\n",
+        "    input = gr.Textbox(show_label=False, type=\"password\", container=False,\n",
+        "                      placeholder=\"●●●●●●●●●●●●●●●●●\")\n",
+        "    input.change(fn=embed_key, inputs=input, outputs=None)\n",
+        "\n",
+        "  with gr.Blocks():\n",
+        "  #########################\n",
+        "  #########Context#########\n",
+        "  #########################\n",
+        "      with gr.Accordion(\"Context section\"):\n",
+        "      ### Should also allow vector stores\n",
+        "          gr.Markdown(\"## Please upload the context document(s) for Oral exam\")\n",
+        "          context_input = gr.File(label=\"Click to upload context file\",\n",
+        "                                  file_count=\"multiple\",\n",
+        "                                  file_types=[\".txt\", \".docx\", \".pdf\"])\n",
+        "          outputs_context=gr.Textbox(label=\"Context\")\n",
+        "          context_input.change(fn=process_file, inputs=context_input, outputs=outputs_context)\n",
+        "          # upload_button = gr.Button(value=\"Show context\")\n",
+        "          # upload_button.click(process_file, context_input, outputs_context)\n",
+        "\n",
+        "  #########################\n",
+        "  ###Question Generation###\n",
+        "  #########################\n",
+        "      with gr.Accordion(\"Question section\"):\n",
+        "          gr.Markdown(\"## Questions\")\n",
+        "          with gr.Row():\n",
+        "            with gr.Column():\n",
+        "              outputs_qa=gr.Textbox(label=\"Generate questions\")\n",
+        "              btn1 = gr.Button(value=\"Generate questions\")\n",
+        "              btn1.click(generate_questions_v2, inputs=outputs_context, outputs=outputs_qa)\n",
+        "\n",
+        "  ######################### Need additional work to include these questions when click button   #########################\n",
+        "            with gr.Column():\n",
+        "              submit_question=gr.Textbox(label=\"Use existing questions\")\n",
+        "              btn4 = gr.Button(value=\"Use these questions\")\n",
+        "              # btn4.click(use_this_question, inputs=outputs_transcribe, outputs=None)\n",
+        "\n",
+        "  #########################\n",
+        "  #######Main Audio########\n",
+        "  #########################\n",
+        "      with gr.Accordion(\"Main audio section\"):\n",
+        "        gr.Markdown(\"## Upload your audio file or start recording\")\n",
+        "        with gr.Column():\n",
+        "          ## uploading files seem not working (don't know why)\n",
+        "          with gr.Row():\n",
+        "              file_input = gr.Audio(label=\"Upload Audio\", source=\"upload\", type=\"filepath\")\n",
+        "              record_inputs = gr.Audio(label=\"Record Audio\", source=\"microphone\", type=\"filepath\")\n",
+        "\n",
+        "          gr.Markdown(\"## Transcribe the audio uploaded or recorded\")\n",
+        "          outputs_transcribe=gr.Textbox(label=\"Transcription\")\n",
+        "\n",
+        "          file_input.change(fn=transcribe, inputs=file_input, outputs=outputs_transcribe)\n",
+        "          record_inputs.change(fn=transcribe, inputs=record_inputs, outputs=outputs_transcribe)\n",
+        "\n",
+        "  #########################\n",
+        "  #######Evaluation########\n",
+        "  #########################\n",
+        "      with gr.Accordion(\"Evaluation section\"):\n",
+        "          gr.Markdown(\"## Evaluation\")\n",
+        "          with gr.Tab(\"General evalution\"):\n",
+        "            evalution=gr.Textbox(label=\"AI Evaluation\")\n",
+        "            btn5 = gr.Button(value=\"Evaluate\")\n",
+        "            btn5.click(ai_evaluate_v2, inputs=[outputs_context, outputs_transcribe, outputs_qa], outputs=evalution)\n",
+        "          with gr.Tab(\"Quantitative evalution\"):\n",
+        "            table_output = gr.Dataframe(label = \"Some kind of evaluation metrics?\")\n",
+        "            btn6 = gr.Button(value=\"Evaluate\")\n",
+        "            btn6.click(ai_evaluate_v2, inputs=[outputs_context, outputs_transcribe, outputs_qa], outputs=table_output)\n",
+        "\n",
+        "  demo.launch()"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 706
+        },
+        "id": "04KxUQgUcTrm",
+        "outputId": "66a9f8c8-36fe-4792-b7d6-3befa6f09269",
+        "cellView": "form",
+        "collapsed": true
+      },
+      "execution_count": 23,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "/usr/local/lib/python3.10/dist-packages/gradio/utils.py:833: UserWarning: Expected 2 arguments for function <function generate_questions_v2 at 0x7aa8748f9bd0>, received 1.\n",
+            "  warnings.warn(\n",
+            "/usr/local/lib/python3.10/dist-packages/gradio/utils.py:837: UserWarning: Expected at least 2 arguments for function <function generate_questions_v2 at 0x7aa8748f9bd0>, received 1.\n",
+            "  warnings.warn(\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Colab notebook detected. To show errors in colab notebook, set debug=True in launch()\n",
+            "Note: opening Chrome Inspector may crash demo inside Colab notebooks.\n",
+            "\n",
+            "To create a public link, set `share=True` in `launch()`.\n"
+          ]
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "<IPython.core.display.Javascript object>"
+            ],
+            "application/javascript": [
+              "(async (port, path, width, height, cache, element) => {\n",
+              "                        if (!google.colab.kernel.accessAllowed && !cache) {\n",
+              "                            return;\n",
+              "                        }\n",
+              "                        element.appendChild(document.createTextNode(''));\n",
+              "                        const url = await google.colab.kernel.proxyPort(port, {cache});\n",
+              "\n",
+              "                        const external_link = document.createElement('div');\n",
+              "                        external_link.innerHTML = `\n",
+              "                            <div style=\"font-family: monospace; margin-bottom: 0.5rem\">\n",
+              "                                Running on <a href=${new URL(path, url).toString()} target=\"_blank\">\n",
+              "                                    https://localhost:${port}${path}\n",
+              "                                </a>\n",
+              "                            </div>\n",
+              "                        `;\n",
+              "                        element.appendChild(external_link);\n",
+              "\n",
+              "                        const iframe = document.createElement('iframe');\n",
+              "                        iframe.src = new URL(path, url).toString();\n",
+              "                        iframe.height = height;\n",
+              "                        iframe.allow = \"autoplay; camera; microphone; clipboard-read; clipboard-write;\"\n",
+              "                        iframe.width = width;\n",
+              "                        iframe.style.border = 0;\n",
+              "                        element.appendChild(iframe);\n",
+              "                    })(7863, \"/\", \"100%\", 500, false, window.element)"
+            ]
+          },
+          "metadata": {}
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "def prompt_select(selection, number, length):\n",
+        "  if selection == \"Random\":\n",
+        "    prompt = f\"Please design a {number} question quiz based on the context provided and the inputted learning objectives (if applicable). The types of questions should be randomized (including multiple choice, short answer, true/false, short answer, etc.).  Provide one question at a time, and wait for my response before providing me with feedback. Again, while the quiz may ask for multiple questions, you should only provide 1 question in you initial response. Do not include the answer in your response. If I get an answer wrong, provide me with an explanation of why it was incorrect, and then give me additional chances to respond until I get the correct choice. Explain why the correct choice is right.\"\n",
+        "  elif selection == \"Fill in the Blank\":\n",
+        "    prompt = f\"Create a {number} question fill in the blank quiz refrencing the context provided. The quiz should reflect the learning objectives (if inputted). The 'blank' part of the question should appear as '________'. The answers should reflect what word(s) should go in the blank an accurate statement. An example is the follow: 'The author of the article is ______.' The question should be a statement. Provide one question at a time, and wait for my response before providing me with feedback. Again, while the quiz may ask for multiple questions, you should only provide ONE question in you initial response. Do not include the answer in your response. If I get an answer wrong, provide me with an explanation of why it was incorrect,and then give me additional chances to respond until I get the correct choice. Explain why the correct choice is right.\"\n",
+        "  elif selection == \"Short Answer\":\n",
+        "    prompt = f\"Please design a {number} question quiz about which reflects the learning objectives (if inputted).  The questions should be short answer. Expect the correct answers to be {length} sentences long. Provide one question at a time, and wait for my response before providing me with feedback. Again, while the quiz may ask for multiple questions, you should only provide ONE question in you initial response. Do not include the answer in your response. If I get an answer wrong, provide me with an explanation of why it was incorrect, and then give me additional chances to respond until I get the correct choice. Explain why the correct answer is right.\"\n",
+        "  else:\n",
+        "    prompt = f\"Please design a {number} question {selection.lower()} quiz based on the context provided and the inputted learning objectives (if applicable). Provide one question at a time, and wait for my response before providing me with feedback. Again, while the quiz may ask for multiple questions, you should only provide 1 question in you initial response. Do not include the answer in your response. If I get an answer wrong, provide me with an explanation of why it was incorrect, and then give me additional chances to respond until I get the correct choice. Explain why the correct choice is right.\"\n",
+        "  return prompt\n",
+        "\n",
+        "\n",
+        "# Function to save prompts (premade or custom) and return in the user input box in the chatbot`\n",
+        "saved_text = \"\"\n",
+        "def save_text(text):\n",
+        "    global saved_text\n",
+        "    saved_text = text\n",
+        "\n",
+        "def return_text():\n",
+        "    # Return the saved text\n",
+        "    return saved_text"
+      ],
+      "metadata": {
+        "id": "wF80F1wU80rU"
+      },
+      "execution_count": 14,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "### Baseline Functionality V3"
+      ],
+      "metadata": {
+        "id": "F5-Ja2evCE4X"
+      }
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "Updated Question Selection and Chatbot Feature"
+      ],
+      "metadata": {
+        "id": "rr8YlzcJCKv4"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "with gr.Blocks() as demo:\n",
+        "  gr.Markdown(\"# Oral Exam App\")\n",
+        "  gr.Markdown(\"## OpenAI API key\")\n",
+        "  with gr.Box():\n",
+        "    gr.HTML(\"\"\"Embed your OpenAI API key below; if you haven't created one already, visit\n",
+        "      platform.openai.com/account/api-keys\n",
+        "    to sign up for an account and get your personal API key\"\"\",\n",
+        "            elem_classes=\"textbox_label\")\n",
+        "    input = gr.Textbox(show_label=False, type=\"password\", container=False,\n",
+        "                      placeholder=\"●●●●●●●●●●●●●●●●●\")\n",
+        "    input.change(fn=embed_key, inputs=input, outputs=None)\n",
+        "\n",
+        "  with gr.Blocks():\n",
+        "  #########################\n",
+        "  #########Context#########\n",
+        "  #########################\n",
+        "      with gr.Accordion(\"Context section\"):\n",
+        "      ### Should also allow vector stores\n",
+        "          gr.Markdown(\"## Please upload the context document(s) for Oral exam\")\n",
+        "          context_input = gr.File(label=\"Click to upload context file\",\n",
+        "                                  file_count=\"multiple\",\n",
+        "                                  file_types=[\".txt\", \".docx\", \".pdf\"])\n",
+        "          outputs_context=gr.Textbox(label=\"Context\")\n",
+        "          context_input.change(fn=process_file, inputs=context_input, outputs=outputs_context)\n",
+        "          # upload_button = gr.Button(value=\"Show context\")\n",
+        "          # upload_button.click(process_file, context_input, outputs_context)\n",
+        "\n",
+        "      with gr.Blocks():\n",
+        "        gr.Markdown(\"\"\"\n",
+        "        ## Generate a Premade Prompt\n",
+        "        Select your type and number of desired questions. Click \"Generate Prompt\" to get your premade prompt,\n",
+        "        and then \"Insert Prompt into Chat\" to copy the text into the chat interface below. \\\n",
+        "        You can also copy the prompt using the icon in the upper right corner and paste directly into the input box when interacting with the model.\n",
+        "        \"\"\")\n",
+        "        with gr.Row():\n",
+        "          with gr.Column():\n",
+        "            question_type = gr.Dropdown([\"Multiple Choice\", \"True or False\", \"Short Answer\", \"Fill in the Blank\", \"Random\"], label=\"Question Type\")\n",
+        "            number_of_questions = gr.Textbox(label=\"Enter desired number of questions\")\n",
+        "            sa_desired_length = gr.Dropdown([\"1-2\", \"3-4\", \"5-6\", \"6 or more\"], label = \"For short answer questions only, choose the desired sentence length for answers. The default value is 1-2 sentences.\")\n",
+        "          with gr.Column():\n",
+        "            prompt_button = gr.Button(\"Generate Prompt\")\n",
+        "            premade_prompt_output = gr.Textbox(label=\"Generated prompt (save or copy)\", show_copy_button=True)\n",
+        "            prompt_button.click(prompt_select,\n",
+        "                            inputs=[question_type, number_of_questions, sa_desired_length],\n",
+        "                            outputs=premade_prompt_output)\n",
+        "  ########################\n",
+        "  ##Question Generation###\n",
+        "  ########################\n",
+        "      with gr.Accordion(\"Question section\"):\n",
+        "          gr.Markdown(\"## Questions\")\n",
+        "          with gr.Row():\n",
+        "            with gr.Column():\n",
+        "              outputs_qa=gr.Textbox(label=\"Generate questions\")\n",
+        "              btn1 = gr.Button(value=\"Generate questions\")\n",
+        "              btn1.click(generate_questions_v2, inputs=[outputs_context, premade_prompt_output], outputs=outputs_qa)\n",
+        "\n",
+        "  ######################### Need additional work to include these questions when click button   #########################\n",
+        "            with gr.Column():\n",
+        "              submit_question=gr.Textbox(label=\"Use existing questions\")\n",
+        "              btn4 = gr.Button(value=\"Use these questions\")\n",
+        "              # btn4.click(use_this_question, inputs=outputs_transcribe, outputs=None)\n",
+        "\n",
+        "  #########################\n",
+        "  #######Main Audio########\n",
+        "  #########################\n",
+        "      with gr.Accordion(\"Main audio section\"):\n",
+        "        gr.Markdown(\"## Upload your audio file or start recording\")\n",
+        "        with gr.Column():\n",
+        "          ## uploading files seem not working (don't know why)\n",
+        "          with gr.Row():\n",
+        "              file_input = gr.Audio(label=\"Upload Audio\", source=\"upload\", type=\"filepath\")\n",
+        "              record_inputs = gr.Audio(label=\"Record Audio\", source=\"microphone\", type=\"filepath\")\n",
+        "\n",
+        "          gr.Markdown(\"## Transcribe the audio uploaded or recorded\")\n",
+        "          outputs_transcribe=gr.Textbox(label=\"Transcription\")\n",
+        "\n",
+        "          file_input.change(fn=transcribe, inputs=file_input, outputs=outputs_transcribe)\n",
+        "          record_inputs.change(fn=transcribe, inputs=record_inputs, outputs=outputs_transcribe)\n",
+        "\n",
+        "  #########################\n",
+        "  #######Evaluation########\n",
+        "  #########################\n",
+        "      with gr.Accordion(\"Evaluation section\"):\n",
+        "          gr.Markdown(\"## Evaluation\")\n",
+        "          with gr.Tab(\"General evalution\"):\n",
+        "            evalution=gr.Textbox(label=\"AI Evaluation\")\n",
+        "            btn5 = gr.Button(value=\"Evaluate\")\n",
+        "            btn5.click(ai_evaluate_v2, inputs=[outputs_context, outputs_transcribe, outputs_qa], outputs=evalution)\n",
+        "          with gr.Tab(\"Quantitative evalution\"):\n",
+        "            table_output = gr.Dataframe(label = \"Some kind of evaluation metrics?\")\n",
+        "            btn6 = gr.Button(value=\"Evaluate\")\n",
+        "            btn6.click(ai_evaluate_v2, inputs=[outputs_context, outputs_transcribe, outputs_qa], outputs=table_output)\n",
+        "\n",
+        "\n",
+        "      # Chatbot (https://gradio.app/creating-a-chatbot/)\n",
+        "      '''\n",
+        "      with gr.Blocks():\n",
+        "        gr.Markdown(\"\"\"\n",
+        "        ## Chat with the Model\n",
+        "        Click \"Display Prompt\" to display the premade or custom prompt that you created earlier. Then, continue chatting with the model.\n",
+        "        \"\"\")\n",
+        "        with gr.Row():\n",
+        "          show_prompt_block = gr.Button(\"Display Prompt\")\n",
+        "      '''\n",
+        "      gr.Markdown(\"## Chat with the Model\")\n",
+        "      with gr.Row(equal_height=True):\n",
+        "        with gr.Column(scale=2):\n",
+        "          chatbot = gr.Chatbot()\n",
+        "          with gr.Row():\n",
+        "            user_chat_input = gr.Textbox(label=\"User input\", scale=9)\n",
+        "            user_chat_input.submit(return_text, inputs=None, outputs=user_chat_input)\n",
+        "            user_chat_submit = gr.Button(\"Ask/answer model\", scale=1)\n",
+        "            #show_prompt_block.click(return_text, inputs=None, outputs=user_chat_input)\n",
+        "\n",
+        "        # TODO Move the sources so it's displayed to the right of the chat bot,\n",
+        "        # with the sources taking up about 1/3rd of the horizontal space\n",
+        "        # with gr.Box(elem_id=\"sources-container\", scale=1):\n",
+        "        #   # TODO: Display document sources in a nicer format?\n",
+        "        #   gr.HTML(value=\"<h3 id='sources'>Sources</h3>\")\n",
+        "        #   sources_output = []\n",
+        "        #   for i in range(num_sources):\n",
+        "        #     source_elem = gr.HTML(visible=False)\n",
+        "        #     sources_output.append(source_elem)\n",
+        "\n",
+        "demo.launch()"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 616
+        },
+        "id": "Y7-3JFuZ8H5k",
+        "outputId": "ea99ce65-7b79-4d39-dd88-44785b0d6615"
+      },
+      "execution_count": 24,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Colab notebook detected. To show errors in colab notebook, set debug=True in launch()\n",
+            "Note: opening Chrome Inspector may crash demo inside Colab notebooks.\n",
+            "\n",
+            "To create a public link, set `share=True` in `launch()`.\n"
+          ]
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "<IPython.core.display.Javascript object>"
+            ],
+            "application/javascript": [
+              "(async (port, path, width, height, cache, element) => {\n",
+              "                        if (!google.colab.kernel.accessAllowed && !cache) {\n",
+              "                            return;\n",
+              "                        }\n",
+              "                        element.appendChild(document.createTextNode(''));\n",
+              "                        const url = await google.colab.kernel.proxyPort(port, {cache});\n",
+              "\n",
+              "                        const external_link = document.createElement('div');\n",
+              "                        external_link.innerHTML = `\n",
+              "                            <div style=\"font-family: monospace; margin-bottom: 0.5rem\">\n",
+              "                                Running on <a href=${new URL(path, url).toString()} target=\"_blank\">\n",
+              "                                    https://localhost:${port}${path}\n",
+              "                                </a>\n",
+              "                            </div>\n",
+              "                        `;\n",
+              "                        element.appendChild(external_link);\n",
+              "\n",
+              "                        const iframe = document.createElement('iframe');\n",
+              "                        iframe.src = new URL(path, url).toString();\n",
+              "                        iframe.height = height;\n",
+              "                        iframe.allow = \"autoplay; camera; microphone; clipboard-read; clipboard-write;\"\n",
+              "                        iframe.width = width;\n",
+              "                        iframe.style.border = 0;\n",
+              "                        element.appendChild(iframe);\n",
+              "                    })(7864, \"/\", \"100%\", 500, false, window.element)"
+            ]
+          },
+          "metadata": {}
+        },
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": []
+          },
+          "metadata": {},
+          "execution_count": 24
+        }
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "### What's left\n",
+        "- vector store (link) upload\n",
+        "- submit question section need to be linked with ai_evaluate function"
+      ],
+      "metadata": {
+        "id": "g2EVIogW69Fd"
+      }
+    }
+  ]
+}

lo-achievement/UI_design_oral_exam_chatbot.ipynb

@@ -0,0 +1,1004 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "view-in-github",
+        "colab_type": "text"
+      },
+      "source": [
+        "<a href=\"https://colab.research.google.com/github/vanderbilt-data-science/lo-achievement/blob/124-implement-baseline-functionality-for-oral-exam-module/UI_design_oral_exam_chatbot.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "PIbogPXyM0wr"
+      },
+      "source": [
+        "# Project IO Achievement - UI Design (Oral Exam)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "x_Vp8SiKM4p1"
+      },
+      "source": [
+        "## Problem Definition\n",
+        "\n",
+        "The v1 functionality for the Oral Exam module requires the following:\n",
+        "\n",
+        "1. Upload or generation of questions: either the user should upload a set of questions or we should allow the model to generate the questions. The user should pick or it should be inherent if there is no upload of questions. Note that we must also allow for context to be uploaded (vector store, vector store link, specific documents)\n",
+        "2. The model should prompt the user with a question and pause.\n",
+        "The user should respond by audio.\n",
+        "3. This should continue on until some final point where the exam is over.\n",
+        "\n",
+        "Then:\n",
+        "\n",
+        "1. We should use Whisper to do the transcription, and\n",
+        "2. Send the transcription, questions, and context for GPT4 for evaluation\n",
+        "Return the evaluation.\n",
+        "3. This will primarily be work on a user interface."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "o_60X8H3NEne"
+      },
+      "source": [
+        "## Libraries\n",
+        "\n",
+        "This section will install and import some important libraries such as Langchain, openai, Gradio, and so on"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 1,
+      "metadata": {
+        "id": "pxcqXgg2aAN7"
+      },
+      "outputs": [],
+      "source": [
+        "# install libraries here\n",
+        "# -q flag for \"quiet\" install\n",
+        "%%capture\n",
+        "!pip install -q langchain\n",
+        "!pip install -q openai\n",
+        "!pip install -q gradio\n",
+        "# !pip install -q datasets\n",
+        "!pip install -q torchaudio\n",
+        "!pip install -q git+https://github.com/openai/whisper.git\n",
+        "!pip install -q docx\n",
+        "!pip install -q PyPDF2\n",
+        "!pip install -q python-docx"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 2,
+      "metadata": {
+        "id": "pEjM1tLsMZBq"
+      },
+      "outputs": [],
+      "source": [
+        "# import libraries here\n",
+        "from langchain.llms import OpenAI\n",
+        "from langchain.prompts import PromptTemplate\n",
+        "from langchain.document_loaders import TextLoader\n",
+        "from langchain.indexes import VectorstoreIndexCreator\n",
+        "from langchain import ConversationChain, LLMChain, PromptTemplate\n",
+        "from langchain.chat_models import ChatOpenAI\n",
+        "from langchain.memory import ConversationBufferWindowMemory\n",
+        "from langchain.prompts import ChatPromptTemplate\n",
+        "from langchain.text_splitter import CharacterTextSplitter\n",
+        "from langchain.embeddings import OpenAIEmbeddings\n",
+        "import openai\n",
+        "import os\n",
+        "from getpass import getpass\n",
+        "# from IPython.display import display, Javascript, HTML\n",
+        "# from google.colab.output import eval_js\n",
+        "# from base64 import b64decode\n",
+        "# import ipywidgets as widgets\n",
+        "# from IPython.display import clear_output\n",
+        "import time\n",
+        "import requests\n",
+        "# from datasets import load_dataset\n",
+        "# from torchaudio.transforms import Resample\n",
+        "import whisper\n",
+        "import numpy as np\n",
+        "import torch\n",
+        "import librosa\n",
+        "# from datasets import load_dataset\n",
+        "#from jiwer import wer\n",
+        "import pandas as pd\n",
+        "import gradio as gr\n",
+        "from docx import Document\n",
+        "import PyPDF2\n",
+        "from pydub import AudioSegment\n",
+        "import tempfile"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "03KLZGI_a5W5"
+      },
+      "source": [
+        "## API Keys\n",
+        "\n",
+        "Use these cells to load the API keys required for this notebook. The below code cell uses the `getpass` library."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 3,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "5smcWj4DbFgy",
+        "outputId": "6bc91507-cd3c-4808-8976-811d7fc7cb29"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "··········\n"
+          ]
+        }
+      ],
+      "source": [
+        "openai_api_key = getpass()\n",
+        "os.environ[\"OPENAI_API_KEY\"] = openai_api_key\n",
+        "openai.api_key = openai_api_key"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "pMo9x8u4AEV1"
+      },
+      "source": [
+        "## Prompt Design"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 4,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "UgnCZRMhADvo",
+        "outputId": "462e62c7-a618-4549-e651-858514757235"
+      },
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "ChatOpenAI(cache=None, verbose=False, callbacks=None, callback_manager=None, tags=None, metadata=None, client=<class 'openai.api_resources.chat_completion.ChatCompletion'>, model_name='gpt-4', temperature=0.0, model_kwargs={}, openai_api_key='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', openai_api_base='', openai_organization='', openai_proxy='', request_timeout=None, max_retries=6, streaming=False, n=1, max_tokens=None, tiktoken_model_name=None)"
+            ]
+          },
+          "metadata": {},
+          "execution_count": 4
+        }
+      ],
+      "source": [
+        "chat = ChatOpenAI(temperature=0.0, model_name='gpt-4')\n",
+        "chat"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "### Chatbot Prompts"
+      ],
+      "metadata": {
+        "id": "2tTNiyU-ZcDU"
+      }
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 15,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "r-VmK_7vHrmw",
+        "outputId": "8c314a8f-dad5-47b9-ddba-f6009a73b80d"
+      },
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "['history', 'input', 'instruction', 'questions']"
+            ]
+          },
+          "metadata": {},
+          "execution_count": 15
+        }
+      ],
+      "source": [
+        "template_string3 = \"\"\"\n",
+        "Please ask me the following questions in sequence, and after I provide the answer, \\\n",
+        "please give me some feedback. Here is the instruction for feedback: {instruction}. If no instruction is provided, please provide feedback based on your judgement. \\\n",
+        "Just ask me the question, and please do not show any other text (no need for greetings for example) \\\n",
+        "Here are the questions that you can will me: {questions}. \\\n",
+        "Here are the chat history: {history}. \\\n",
+        "{input}\n",
+        "\n",
+        "Once all questions are answered, thank the user and give overall feedback for the question answering part.\n",
+        "\"\"\"\n",
+        "prompt_template3 = ChatPromptTemplate.from_template(template_string3)\n",
+        "prompt_template3.messages[0].prompt.input_variables"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 16,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 72
+        },
+        "id": "XaK7D5B4bMYv",
+        "outputId": "55f495b5-d8d0-4a80-b5c4-128dba34eebe"
+      },
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "'\\nPlease ask me the following questions in sequence, and after I provide the answer, please give me some feedback. Here is the instruction for feedback: {instruction}. If no instruction is provided, please provide feedback based on your judgement. Just ask me the question, and please do not show any other text (no need for greetings for example) Here are the questions that you can will me: {questions}. Here are the chat history: {history}. {input}\\n\\nOnce all questions are answered, thank the user and give overall feedback for the question answering part.\\n'"
+            ],
+            "application/vnd.google.colaboratory.intrinsic+json": {
+              "type": "string"
+            }
+          },
+          "metadata": {},
+          "execution_count": 16
+        }
+      ],
+      "source": [
+        "prompt_template3.messages[0].prompt.template"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "l4o8R5eUE1n8"
+      },
+      "source": [
+        "### Functions"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 7,
+      "metadata": {
+        "id": "ABN0X9xQHeii"
+      },
+      "outputs": [],
+      "source": [
+        "def embed_key(openai_api_key):\n",
+        "  os.environ[\"OPENAI_API_KEY\"] = openai_api_key\n",
+        "\n",
+        "def transcribe(audio_file_path):\n",
+        "  try:\n",
+        "    with open(audio_file_path, \"rb\") as audio_file:\n",
+        "      # Call OpenAI's Whisper model for transcription\n",
+        "      transcript = openai.Audio.transcribe(\"whisper-1\", audio_file)\n",
+        "      transcribed_text = transcript[\"text\"]\n",
+        "      return transcribed_text\n",
+        "  except:\n",
+        "    return \"Your answer will be transcribed here\"\n",
+        "\n",
+        "def process_file(files):\n",
+        "  for file in files:\n",
+        "    try:\n",
+        "        extension = file.name.split('.')[-1].lower()\n",
+        "        if extension == 'docx':\n",
+        "            doc = Document(file.name)\n",
+        "            full_text = []\n",
+        "            for paragraph in doc.paragraphs:\n",
+        "                full_text.append(paragraph.text)\n",
+        "            return '\\n'.join(full_text)\n",
+        "\n",
+        "        elif extension == 'pdf':\n",
+        "            pdf_file = open(file.name, 'rb')\n",
+        "            reader = PyPDF2.PdfReader(pdf_file)\n",
+        "            num_pages = len(reader.pages)\n",
+        "            full_text = []\n",
+        "            for page in range(num_pages):\n",
+        "                page_obj = reader.pages[page]\n",
+        "                full_text.append(page_obj.extract_text())\n",
+        "            pdf_file.close()\n",
+        "            return '\\n'.join(full_text)\n",
+        "\n",
+        "        elif extension == 'txt':\n",
+        "            with open(file.name, 'r') as txt_file:\n",
+        "                full_text = txt_file.read()\n",
+        "            return full_text\n",
+        "\n",
+        "        else:\n",
+        "            return \"Unsupported file type\"\n",
+        "    except FileNotFoundError:\n",
+        "        return \"File not found\"\n",
+        "    except PermissionError:\n",
+        "        return \"Permission denied\"\n",
+        "\n",
+        "def generate_questions(text, prompt):\n",
+        "    test_input1 = question_template.format_messages(\n",
+        "                      context = text,\n",
+        "                      pre_prompt = prompt)\n",
+        "\n",
+        "    response = chat(test_input1)\n",
+        "    return response.content\n",
+        "\n",
+        "\n",
+        "def ai_evaluate(context, audio_transcript, QA, instructions):\n",
+        "    test_input1 = evaluate_template.format_messages(\n",
+        "                      context = context,\n",
+        "                      transcript = audio_transcript,\n",
+        "                      QA = QA,\n",
+        "                      instructions = instructions)\n",
+        "\n",
+        "    response = chat(test_input1)\n",
+        "    return response.content\n",
+        "\n",
+        "def upload_file(files):\n",
+        "    file_paths = [file.name for file in files]\n",
+        "    return file_paths\n",
+        "\n",
+        "def use_these_questions(input):\n",
+        "    return input\n",
+        "\n",
+        "################################\n",
+        "\n",
+        "def add_text(history, text, prompt = template_string3):\n",
+        "    new_history = [(prompt, None)] + history + [(text, None)]\n",
+        "    return new_history, gr.update(value=\"\", interactive=False)\n",
+        "\n",
+        "# def add_file(history, file):\n",
+        "#     history = history + [((file.name,), None)]\n",
+        "#     return history\n",
+        "\n",
+        "\n",
+        "def bot_initialize(input, instruction_feedback, questions_used, history):\n",
+        "\n",
+        "    template_string3 = \"\"\"\n",
+        "    Please ask me the following questions in sequence, and after I provide the answer, \\\n",
+        "    please give me some feedback. Here is the instruction for feedback: {instruction}. If no instruction is provided, please provide feedback based on your judgement. \\\n",
+        "    Here are the questions that you can ask me: {questions}. \\\n",
+        "    Here are the chat history: {history}. \\\n",
+        "    {input} \\\n",
+        "\n",
+        "    *** Remember, just ask me the question, give feedbacks, and ask the next questions. Do not forget to ask the next question after feedbacks. \\\n",
+        "    \"\"\"\n",
+        "    prompt_template3 = ChatPromptTemplate.from_template(template_string3)\n",
+        "\n",
+        "    test_input1 = prompt_template3.format_messages(\n",
+        "                      instruction = instruction_feedback,\n",
+        "                      history = history,\n",
+        "                      questions = questions_used,\n",
+        "                      input = input)\n",
+        "\n",
+        "    response = chat(test_input1)\n",
+        "    return response.content\n",
+        "\n",
+        "# def initialize(instruction_feedback, questions_used, chat_history, ready):\n",
+        "#     test_input1 = prompt_template3.format_messages(\n",
+        "#                       instruction = instruction_feedback,\n",
+        "#                       chat_history = chat_history,\n",
+        "#                       questions = questions_used,\n",
+        "#                       ready = ready)\n",
+        "#     response = chat(test_input1)\n",
+        "#     return response.content\n",
+        "\n",
+        "# def bot(history):\n",
+        "#     response = \"**That's cool!**\"\n",
+        "#     history[-1][1] = \"\"\n",
+        "#     for character in response:\n",
+        "#         history[-1][1] += character\n",
+        "#         time.sleep(0.05)\n",
+        "#         yield history\n",
+        "\n",
+        "def message_and_history(input, instruction_feedback, questions_used, history):\n",
+        "    history = history or []\n",
+        "    s = list(sum(history, ()))\n",
+        "    s.append(input)\n",
+        "    inp = ' '.join(s)\n",
+        "    output = bot_initialize(inp, instruction_feedback, questions_used, history)\n",
+        "    history.append((input, output))\n",
+        "    return history, history\n",
+        "\n",
+        "def prompt_select(selection, number, length):\n",
+        "  if selection == \"Random\":\n",
+        "    prompt = f\"Please design a {number} question quiz based on the context provided and the inputted learning objectives (if applicable).\"\n",
+        "  elif selection == \"Fill in the Blank\":\n",
+        "    prompt = f\"Create a {number} question fill in the blank quiz refrencing the context provided. The quiz should reflect the learning objectives (if inputted). The 'blank' part of the question should appear as '________'. The answers should reflect what word(s) should go in the blank an accurate statement. An example is the follow: 'The author of the article is ______.' The question should be a statement.\"\n",
+        "  elif selection == \"Short Answer\":\n",
+        "    prompt = f\"Please design a {number} question quiz about which reflects the learning objectives (if inputted).  The questions should be short answer. Expect the correct answers to be {length} sentences long.\"\n",
+        "  else:\n",
+        "    prompt = f\"Please design a {number} question {selection.lower()} quiz based on the context provided and the inputted learning objectives (if applicable).\"\n",
+        "  return prompt\n",
+        "\n",
+        "# def prompt_select(selection, number, length):\n",
+        "#   if selection == \"Random\":\n",
+        "#     prompt = f\"Please design a {number} question quiz based on the context provided and the inputted learning objectives (if applicable). The types of questions should be randomized (including multiple choice, short answer, true/false, short answer, etc.).  Provide one question at a time, and wait for my response before providing me with feedback. Again, while the quiz may ask for multiple questions, you should only provide 1 question in you initial response. Do not include the answer in your response. If I get an answer wrong, provide me with an explanation of why it was incorrect, and then give me additional chances to respond until I get the correct choice. Explain why the correct choice is right.\"\n",
+        "#   elif selection == \"Fill in the Blank\":\n",
+        "#     prompt = f\"Create a {number} question fill in the blank quiz refrencing the context provided. The quiz should reflect the learning objectives (if inputted). The 'blank' part of the question should appear as '________'. The answers should reflect what word(s) should go in the blank an accurate statement. An example is the follow: 'The author of the article is ______.' The question should be a statement. Provide one question at a time, and wait for my response before providing me with feedback. Again, while the quiz may ask for multiple questions, you should only provide ONE question in you initial response. Do not include the answer in your response. If I get an answer wrong, provide me with an explanation of why it was incorrect,and then give me additional chances to respond until I get the correct choice. Explain why the correct choice is right.\"\n",
+        "#   elif selection == \"Short Answer\":\n",
+        "#     prompt = f\"Please design a {number} question quiz about which reflects the learning objectives (if inputted).  The questions should be short answer. Expect the correct answers to be {length} sentences long. Provide one question at a time, and wait for my response before providing me with feedback. Again, while the quiz may ask for multiple questions, you should only provide ONE question in you initial response. Do not include the answer in your response. If I get an answer wrong, provide me with an explanation of why it was incorrect, and then give me additional chances to respond until I get the correct choice. Explain why the correct answer is right.\"\n",
+        "#   else:\n",
+        "#     prompt = f\"Please design a {number} question {selection.lower()} quiz based on the context provided and the inputted learning objectives (if applicable). Provide one question at a time, and wait for my response before providing me with feedback. Again, while the quiz may ask for multiple questions, you should only provide 1 question in you initial response. Do not include the answer in your response. If I get an answer wrong, provide me with an explanation of why it was incorrect, and then give me additional chances to respond until I get the correct choice. Explain why the correct choice is right.\"\n",
+        "#   return prompt"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8PzIpcfg4-X0"
+      },
+      "source": [
+        "## Integrate Prompts from LO project"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "E5vWxcm25EAC"
+      },
+      "source": [
+        "### Creating a Chain for Short Answer Generation"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "y95FExV-5IqI"
+      },
+      "source": [
+        "In this example, the context would include the poem \"The Road Not Taken\" by Robert Frost"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 8,
+      "metadata": {
+        "id": "j_qEXDWQ5RSW"
+      },
+      "outputs": [],
+      "source": [
+        "# This is what I used to test the function 'generate_questions_v2'\n",
+        "template_string = \"\"\"\n",
+        "You are a world-class tutor helping students to perform better on oral and written exams though interactive experiences.\"\n",
+        "\n",
+        "The following text should be used as the basis for the instructions which follow: {context} \\\n",
+        "\n",
+        "The following is the guideline for generating the questiion: {pre_prompt} \\\n",
+        "\n",
+        "The output should be formatted as following:\n",
+        "\n",
+        "Question 1: ...\n",
+        "Question 2: ...\n",
+        "Question 3: ...\n",
+        "...\n",
+        "\"\"\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 9,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "yWOM1XdC5UhQ",
+        "outputId": "69d781f7-fb0c-4dde-9085-ddb9908b82af"
+      },
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "['context', 'pre_prompt']"
+            ]
+          },
+          "metadata": {},
+          "execution_count": 9
+        }
+      ],
+      "source": [
+        "question_template = ChatPromptTemplate.from_template(template_string)\n",
+        "question_template.messages[0].prompt.input_variables"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 10,
+      "metadata": {
+        "id": "4Mc1ZC3jaydQ"
+      },
+      "outputs": [],
+      "source": [
+        "# @title\n",
+        "con = \"\"\" Two roads diverged in a yellow wood,\n",
+        "And sorry I could not travel both\n",
+        "And be one traveler, long I stood\n",
+        "And looked down one as far as I could\n",
+        "To where it bent in the undergrowth;\n",
+        "Then took the other, as just as fair,\n",
+        "And having perhaps the better claim,\n",
+        "Because it was grassy and wanted wear;\n",
+        "Though as for that the passing there\n",
+        "Had worn them really about the same,\n",
+        "And both that morning equally lay\n",
+        "In leaves no step had trodden black.\n",
+        "Oh, I kept the first for another day!\n",
+        "Yet knowing how way leads on to way,\n",
+        "I doubted if I should ever come back.\n",
+        "I shall be telling this with a sigh\n",
+        "Somewhere ages and ages hence:\n",
+        "Two roads diverged in a wood, and I—\n",
+        "I took the one less traveled by,\n",
+        "And that has made all the difference.\n",
+        "—-Robert Frost—-\n",
+        "Education Place: http://www.eduplace.com \"\"\"\n",
+        "\n",
+        "pre = \"Please design a 3 question quiz about which reflects the learning objectives (if inputted).  The questions should be short answer. Expect the correct answers to be  sentences long. Provide one question at a time, and wait for my response before providing me with feedback. Again, while the quiz may ask for multiple questions, you should only provide ONE question in you initial response. Do not include the answer in your response. If I get an answer wrong, provide me with an explanation of why it was incorrect, and then give me additional chances to respond until I get the correct choice. Explain why the correct answer is right.\"\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 11,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 36
+        },
+        "id": "KXssPFyEbG3f",
+        "outputId": "8a5389f7-3ec6-4332-d9ae-f45ff5781caf"
+      },
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "'Question 1: What is the main theme of Robert Frost\\'s poem \"The Road Not Taken\"?'"
+            ],
+            "application/vnd.google.colaboratory.intrinsic+json": {
+              "type": "string"
+            }
+          },
+          "metadata": {},
+          "execution_count": 11
+        }
+      ],
+      "source": [
+        "generate_questions(con,pre)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "DMTybR3PVuoC"
+      },
+      "source": [
+        "### Creating a Chain for AI Evaluation"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 12,
+      "metadata": {
+        "id": "Wc-3XAFQVxO_"
+      },
+      "outputs": [],
+      "source": [
+        "template_evaluation = \"\"\"\n",
+        "Given\n",
+        "1. The follwing context of the oral exam/presentation: {context} \\\n",
+        "\n",
+        "2. The answer from the student: {transcript} \\\n",
+        "\n",
+        "3. The Questions asked to the student and student answers {QA} \\\n",
+        "\n",
+        "Please evaluate the students performance based on {instructions} \\\n",
+        "\n",
+        "If no instruction is provided, you can evaluate based on your judgement of the students performance.\n",
+        "\n",
+        "\"\"\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 13,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "FZXeYNSVVy5g",
+        "outputId": "94a1a4be-03a6-4c20-97b8-4333910991fa"
+      },
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "['QA', 'context', 'instructions', 'transcript']"
+            ]
+          },
+          "metadata": {},
+          "execution_count": 13
+        }
+      ],
+      "source": [
+        "# @title\n",
+        "evaluate_template = ChatPromptTemplate.from_template(template_evaluation)\n",
+        "evaluate_template.messages[0].prompt.input_variables"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "a3WUL_hFyMkr"
+      },
+      "source": [
+        "### Test process_file"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 157
+        },
+        "id": "LJX1AKTMyVm8",
+        "outputId": "24f8de7a-f456-47bc-b3b5-0284bbd7076f"
+      },
+      "outputs": [
+        {
+          "data": {
+            "application/vnd.google.colaboratory.intrinsic+json": {
+              "type": "string"
+            },
+            "text/plain": [
+              "\"\\ufeffHello,\\n\\n\\nWe are so excited for this semester’s partnership with Data Science Institute and Next Steps at Vanderbilt. Jonathan Wade will be interning Mondays and Wednesdays 2-5 pm and Fridays 12-4 pm starting Monday, January 31st. Jessica will be job coaching on Mondays and Wednesdays from 2-5 pm. It also used to be on Fridays from 2-4 pm but not anymore.\\n\\n\\nBelow is important information and reminders:\\n\\n\\nLocation: 1400 18th Ave S Building, Suite 2000, Nashville, TN 37212\\n\\n\\nDress: Business casual attire (This includes items such as dress pants, khakis, polos and dress shirts)\\n\\n\\nImportant Dates:\\n\\n\\nVanderbilt University is off for Spring Break March 5th - March 13th. No internships will take place these days.\\n\\n\\nAll internships end by Thursday, April 28th.\\n\\n\\nImportant COVID-19 Information: Attached is a document outlining the COVID-19 guidelines all Vanderbilt University students must follow, including Next Steps interns and job coaches while at internship sites. Please note these may change given the evolving nature of the pandemic and any changes will be communicated to internship sites, interns, and job coaches as needed.\\n\\n\\nCareer Development Resource Guide: I am also attaching the Next Steps Career Development Resource guide that outlines student expectations and provides helpful information and resources for site supervisors.\\n\\n\\nInternship Coordinator: Lynda Tricia is the coordinator for this internship and is the main point of contact for any questions or concerns that arise.\\n\\n\\nFinally, below you will find everyone's contact information for your convenience.\\n\\n\\nContacts:\\n\\n\\nIntern: Jonathan Wade, [email protected], 613-472-3867\\n\\n\\nSupervisor: Ruben Miller, [email protected], 216-574-3176\\n\\n\\nJob Coach: Jessica Cho, [email protected], 615-999-1134\\n\\n\\nInternship Coordinator: Lynda Tricia, [email protected], 606-415-9999\\n\\n\\nPlease let us know if there are any questions. Thank you!\\n\\n\\n\\n\\nNext Steps at Vanderbilt - Safety Guidelines for Internships\\nMore information on Vanderbilt’s Health and Safety Protocols can be found here: https://www.vanderbilt.edu/coronavirus/community/undergraduate-students/\\nMasks: All Next Steps interns and job coaches must wear masks indoors at internships even if the jobsite does not require masks.\\nInterns and job coaches should have a well-fitted mask that completely covers your nose and mouth, preferably a KN95, KF94 or FFP2 version.\\nLunch Breaks: If an intern or job coach needs a lunch break, they can remove their mask just when they eat or drink. They must be physically distanced from other co-workers when eating. \\nSymptom Monitoring: All interns and job coaches must be free of ANY symptoms related to COVID-19 to go to the internship. If an intern or job coach has symptoms, they should stay home, notify the Next Steps staff and internship supervisor, and get tested at Vanderbilt Student Health or with their medical provider.\\nAccording to the CDC, symptoms may appear 2 to 14 days after exposure to the virus. These include:\\n* Fever or chills\\n* Cough\\n* Shortness of breath or difficulty breathing\\n* Fatigue\\n* Muscle or body aches\\n* Headache\\n* New loss of taste or smell\\n* Sore throat\\n* Congestion or runny nose\\n* Nausea or vomiting\\n* Diarrhea\\n\\n\\nIf an intern or job coach tests positive: If an intern or job coach receives a COVID-19 positive test result, regardless of vaccination status, they should complete the following webform. The webform goes directly to the Vanderbilt Command Center.\\nThe intern or job coach will receive direct communication from the Command Center about their isolation (if they tested positive) or quarantine period (if considered a close contact) and will be instructed to contact Student Health or Occupational Health if they develop symptoms.\\nClose Contact/Quarantine: Interns or job coaches who are a close contact of someone who tests positive should complete the following webform. The webform goes directly to the Command Center.\\n* Close contacts who are unvaccinated will quarantine for 10 days based on CDC guidance.\\no Additional requirements are in place for days 10 to 14 following exposure including:\\n* For days 10 to 14 after last exposure, unvaccinated Vanderbilt community members identified as close contacts must not unmask at any time in public.\\n* Individuals should eat alone or complete any activities alone that require removing a mask in a private space during those four days between day 10-14.\\n* Close contacts who are vaccinated and asymptomatic will not have to quarantine but are recommended to monitor their symptoms and to get a COVID-19 test 5-7 days after last exposure. If asymptomatic, testing can be done at the VU testing center. If individuals develop symptoms, they should test at Student Health, Occupational Health or VUMC or other testing location in the community.\\n* Close contacts who are vaccinated and symptomatic may have to quarantine based on severity of symptoms and specific living situations. This determination will be made by their medical provider in consultation with the Command Center.\\nJob Coaching Supports: If a job coach tests positive and is unable to provide on-site supports, Next Steps staff will follow the procedures outlined below.\\n1. Identify another job coach or Next Steps staff member who can provide job coaching on-site during some of the student’s internship hours.\\n2. Identify another job coach or staff member who can check-in virtually with the student and supervisor during their shift. \\n3. Or, work with the student and supervisor to ensure natural supports are in place if the student must work without support of a job coach during that time period.\""
+            ]
+          },
+          "execution_count": 69,
+          "metadata": {},
+          "output_type": "execute_result"
+        }
+      ],
+      "source": [
+        "# Might need some way to make pdf file to load more readable\n",
+        "# process_file('/content/instrutor_note.docx')\n",
+        "# process_file('/content/Big Data & Economics.pdf')\n",
+        "# process_file('/content/Big Data & Economics.pdf')\n",
+        "process_file('/content/Anonymized Job Coach QA Test Doc (1).txt')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "M6IzVTjz5cex"
+      },
+      "source": [
+        "## UI Design\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "u2SY4Akt_t8h"
+      },
+      "source": [
+        "### Chatbot V2"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 17,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 853
+        },
+        "id": "6ENnsKlD_uOC",
+        "outputId": "7b694aa8-e314-4502-a590-3245ebc3e1e0"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Colab notebook detected. This cell will run indefinitely so that you can see errors and logs. To turn off, set debug=False in launch().\n",
+            "Note: opening Chrome Inspector may crash demo inside Colab notebooks.\n",
+            "\n",
+            "To create a public link, set `share=True` in `launch()`.\n"
+          ]
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "<IPython.core.display.Javascript object>"
+            ],
+            "application/javascript": [
+              "(async (port, path, width, height, cache, element) => {\n",
+              "                        if (!google.colab.kernel.accessAllowed && !cache) {\n",
+              "                            return;\n",
+              "                        }\n",
+              "                        element.appendChild(document.createTextNode(''));\n",
+              "                        const url = await google.colab.kernel.proxyPort(port, {cache});\n",
+              "\n",
+              "                        const external_link = document.createElement('div');\n",
+              "                        external_link.innerHTML = `\n",
+              "                            <div style=\"font-family: monospace; margin-bottom: 0.5rem\">\n",
+              "                                Running on <a href=${new URL(path, url).toString()} target=\"_blank\">\n",
+              "                                    https://localhost:${port}${path}\n",
+              "                                </a>\n",
+              "                            </div>\n",
+              "                        `;\n",
+              "                        element.appendChild(external_link);\n",
+              "\n",
+              "                        const iframe = document.createElement('iframe');\n",
+              "                        iframe.src = new URL(path, url).toString();\n",
+              "                        iframe.height = height;\n",
+              "                        iframe.allow = \"autoplay; camera; microphone; clipboard-read; clipboard-write;\"\n",
+              "                        iframe.width = width;\n",
+              "                        iframe.style.border = 0;\n",
+              "                        element.appendChild(iframe);\n",
+              "                    })(7860, \"/\", \"100%\", 500, false, window.element)"
+            ]
+          },
+          "metadata": {}
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "/usr/local/lib/python3.10/dist-packages/gradio/processing_utils.py:188: UserWarning: Trying to convert audio automatically from int32 to 16-bit int format.\n",
+            "  warnings.warn(warning.format(data.dtype))\n",
+            "/usr/local/lib/python3.10/dist-packages/gradio/processing_utils.py:188: UserWarning: Trying to convert audio automatically from int32 to 16-bit int format.\n",
+            "  warnings.warn(warning.format(data.dtype))\n",
+            "/usr/local/lib/python3.10/dist-packages/gradio/processing_utils.py:188: UserWarning: Trying to convert audio automatically from int32 to 16-bit int format.\n",
+            "  warnings.warn(warning.format(data.dtype))\n",
+            "/usr/local/lib/python3.10/dist-packages/gradio/processing_utils.py:188: UserWarning: Trying to convert audio automatically from int32 to 16-bit int format.\n",
+            "  warnings.warn(warning.format(data.dtype))\n",
+            "/usr/local/lib/python3.10/dist-packages/gradio/processing_utils.py:188: UserWarning: Trying to convert audio automatically from int32 to 16-bit int format.\n",
+            "  warnings.warn(warning.format(data.dtype))\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Keyboard interruption in main thread... closing server.\n"
+          ]
+        },
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "'\\nWhat Are the Different Types of Machine Learning?\\nHow Do You Handle Missing or Corrupted Data in a Dataset?\\nHow Can You Choose a Classifier Based on a Training Set Data Size?\\nExplain the Confusion Matrix with Respect to Machine Learning Algorithms.\\nWhat Are the Differences Between Machine Learning and Deep Learning\\n'"
+            ],
+            "application/vnd.google.colaboratory.intrinsic+json": {
+              "type": "string"
+            }
+          },
+          "metadata": {},
+          "execution_count": 17
+        }
+      ],
+      "source": [
+        "with gr.Blocks(theme=gr.themes.Monochrome()) as demo:\n",
+        "  gr.Markdown(\"# Oral Exam App\")\n",
+        "  gr.Markdown(\"## OpenAI API key\")\n",
+        "  with gr.Box():\n",
+        "    gr.HTML(\"\"\"Embed your OpenAI API key below; if you haven't created one already, visit\n",
+        "      platform.openai.com/account/api-keys\n",
+        "    to sign up for an account and get your personal API key\"\"\",\n",
+        "            elem_classes=\"textbox_label\")\n",
+        "    input = gr.Textbox(show_label=False, type=\"password\", container=False,\n",
+        "                      placeholder=\"●●●●●●●●●●●●●●●●●\")\n",
+        "    input.change(fn=embed_key, inputs=input, outputs=None)\n",
+        "\n",
+        "  with gr.Blocks():\n",
+        "  #########################\n",
+        "  #########Context#########\n",
+        "  #########################\n",
+        "      with gr.Accordion(\"Context section\"):\n",
+        "      ### Should also allow vector stores\n",
+        "          gr.Markdown(\"## Please upload the context document(s) for Oral exam\")\n",
+        "          context_input = gr.File(label=\"Click to upload context file\",\n",
+        "                                  file_count=\"multiple\",\n",
+        "                                  file_types=[\".txt\", \".docx\", \".pdf\"])\n",
+        "          outputs_context=gr.Textbox(label=\"Context\")\n",
+        "          context_input.change(fn=process_file, inputs=context_input, outputs=outputs_context)\n",
+        "          # upload_button = gr.Button(value=\"Show context\")\n",
+        "          # upload_button.click(process_file, context_input, outputs_context)\n",
+        "\n",
+        "      with gr.Blocks():\n",
+        "        gr.Markdown(\"\"\"\n",
+        "        ## Generate a Premade Prompt\n",
+        "        Select your type and number of desired questions. Click \"Generate Prompt\" to get your premade prompt,\n",
+        "        and then \"Insert Prompt into Chat\" to copy the text into the chat interface below. \\\n",
+        "        You can also copy the prompt using the icon in the upper right corner and paste directly into the input box when interacting with the model.\n",
+        "        \"\"\")\n",
+        "        with gr.Row():\n",
+        "          with gr.Column():\n",
+        "            question_type = gr.Dropdown([\"Multiple Choice\", \"True or False\", \"Short Answer\", \"Fill in the Blank\", \"Random\"], label=\"Question Type\")\n",
+        "            number_of_questions = gr.Textbox(label=\"Enter desired number of questions\")\n",
+        "            sa_desired_length = gr.Dropdown([\"1-2\", \"3-4\", \"5-6\", \"6 or more\"], label = \"For short answer questions only, choose the desired sentence length for answers. The default value is 1-2 sentences.\")\n",
+        "          with gr.Column():\n",
+        "            prompt_button = gr.Button(\"Generate Prompt\")\n",
+        "            premade_prompt_output = gr.Textbox(label=\"Generated prompt (save or copy)\", show_copy_button=True)\n",
+        "            prompt_button.click(prompt_select,\n",
+        "                            inputs=[question_type, number_of_questions, sa_desired_length],\n",
+        "                            outputs=premade_prompt_output)\n",
+        "\n",
+        "  #########################\n",
+        "  #######Main Audio########\n",
+        "  #########################\n",
+        "      with gr.Accordion(\"Main audio section\"):\n",
+        "        gr.Markdown(\"## Upload your audio file or start recording\")\n",
+        "\n",
+        "        with gr.Column():\n",
+        "          with gr.Row():\n",
+        "              file_input = gr.Audio(label=\"Upload Audio\", source=\"upload\", type=\"filepath\")\n",
+        "              record_inputs = gr.Audio(label=\"Record Audio\", source=\"microphone\", type=\"filepath\")\n",
+        "\n",
+        "          gr.Markdown(\"## Transcribe the audio uploaded or recorded\")\n",
+        "          outputs_transcribe=gr.Textbox(label=\"Transcription\")\n",
+        "\n",
+        "          file_input.change(fn=transcribe, inputs=file_input, outputs=outputs_transcribe)\n",
+        "          record_inputs.change(fn=transcribe, inputs=record_inputs, outputs=outputs_transcribe)\n",
+        "\n",
+        "  # #########################\n",
+        "  # ###Question Generation###\n",
+        "  # #########################\n",
+        "  #     with gr.Accordion(\"Question section\"):\n",
+        "  #         gr.Markdown(\"## Questions\")\n",
+        "  #         with gr.Column():\n",
+        "  #           outputs_qa=gr.Textbox(label=\"Generate questions or Use your own questions\")\n",
+        "  #           btn3 = gr.Button(value=\"Generate questions\")\n",
+        "  #           btn3.click(generate_questions, inputs=context_input, outputs=outputs_qa)\n",
+        "\n",
+        "\n",
+        "  ########################\n",
+        "  ##Question Generation###\n",
+        "  ########################\n",
+        "      with gr.Accordion(\"Question section\"):\n",
+        "          gr.Markdown(\"## Questions\")\n",
+        "          with gr.Row():\n",
+        "            with gr.Column():\n",
+        "              outputs_qa=gr.Textbox(label=\"Generate questions or Use your own questions\")\n",
+        "              btn1 = gr.Button(value=\"Generate questions\")\n",
+        "              btn1.click(generate_questions, inputs=[outputs_context, premade_prompt_output], outputs=outputs_qa)\n",
+        "\n",
+        "            # with gr.Column():\n",
+        "            #   submit_question=gr.Textbox(label=\"Use existing questions\")\n",
+        "            #   btn4 = gr.Button(value=\"Use these questions\")\n",
+        "              # btn4.click(use_this_question, inputs=outputs_transcribe, outputs=None)\n",
+        "\n",
+        "\n",
+        "  #########################\n",
+        "  #######Instruction#######\n",
+        "  #########################\n",
+        "          instruction_qa_input = gr.File(label=\"Click to upload instruction file\",\n",
+        "                                  file_count=\"multiple\",\n",
+        "                                  file_types=[\".txt\", \".docx\", \".pdf\"])\n",
+        "          instruction_qa=gr.Textbox(label=\"Or please enter the instruction for question/answering section\")\n",
+        "          instruction_qa.change(fn=process_file, inputs=context_input, outputs=outputs_context)\n",
+        "\n",
+        "\n",
+        "  #########################\n",
+        "  #########Audio QA########\n",
+        "  #########################\n",
+        "      with gr.Accordion(\"Audio QA section\"):\n",
+        "          gr.Markdown(\"## Question answering\")\n",
+        "          gr.Markdown(\"### When you are ready to answer questions, press the 'I am ready' button\")\n",
+        "          ##### This may be iterative\n",
+        "          chatbot = gr.Chatbot([],\n",
+        "                                  elem_id=\"chatbot\",\n",
+        "                                  height=300)\n",
+        "          state = gr.State()\n",
+        "          message = gr.Textbox(show_label=False,\n",
+        "                              placeholder=\"Your answer will be transcribed here\",\n",
+        "                              container=False)\n",
+        "          ready_button = gr.Button(value=\"I am ready\")\n",
+        "          ready_button.click(message_and_history, inputs=[message, instruction_qa, outputs_qa, state], outputs=[chatbot, state])\n",
+        "\n",
+        "          hidden = gr.Textbox(visible = False)\n",
+        "          btn_record = gr.Audio(label=\"Record Audio\", source=\"microphone\", type=\"filepath\")\n",
+        "          btn_record.change(fn=transcribe, inputs=btn_record, outputs=message)\n",
+        "          btn_record.clear(use_these_questions, inputs = hidden, outputs = message)\n",
+        "\n",
+        "          submit = gr.Button(\"Submit\")\n",
+        "          submit.click(message_and_history,\n",
+        "                      inputs=[message, instruction_qa, outputs_qa, state],\n",
+        "                      outputs=[chatbot, state])\n",
+        "\n",
+        "          message_records = gr.Textbox(show_label=False,\n",
+        "                              container=False)\n",
+        "          show_records = gr.Button(\"Show QA history\")\n",
+        "          show_records.click(use_these_questions,\n",
+        "                      inputs=state,\n",
+        "                      outputs=message_records)\n",
+        "\n",
+        "  #########################\n",
+        "  #######Evaluation########\n",
+        "  #########################\n",
+        "      with gr.Accordion(\"Evaluation section\"):\n",
+        "          gr.Markdown(\"## Evaluation\")\n",
+        "          with gr.Tab(\"General evalution\"):\n",
+        "            evalution=gr.Textbox(label=\"AI Evaluation\")\n",
+        "            btn5 = gr.Button(value=\"Evaluate\")\n",
+        "            btn5.click(ai_evaluate, inputs=[outputs_context, outputs_transcribe, message_records, instruction_qa], outputs=evalution)\n",
+        "          with gr.Tab(\"Quantitative evalution\"):\n",
+        "            table_output = gr.Dataframe(label = \"Some kind of evaluation metrics?\")\n",
+        "            btn6 = gr.Button(value=\"Evaluate\")\n",
+        "            # btn6.click(ai_evaluate, inputs=[outputs_context, message_records, outputs_qa], outputs=table_output)\n",
+        "\n",
+        "  # demo.launch()\n",
+        "  # demo.launch(share=True)\n",
+        "  demo.launch(debug=True)\n",
+        "\n",
+        "'''\n",
+        "What Are the Different Types of Machine Learning?\n",
+        "How Do You Handle Missing or Corrupted Data in a Dataset?\n",
+        "How Can You Choose a Classifier Based on a Training Set Data Size?\n",
+        "Explain the Confusion Matrix with Respect to Machine Learning Algorithms.\n",
+        "What Are the Differences Between Machine Learning and Deep Learning\n",
+        "'''"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "g2EVIogW69Fd"
+      },
+      "source": [
+        "## What's left\n",
+        "- vector store (link) upload\n",
+        "- how to not show the warning when transcribing\n",
+        "- better prompt for evaluation\n",
+        "- try ChatInterface of Gradio"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [],
+      "metadata": {
+        "id": "-YwOAtNANrx_"
+      },
+      "execution_count": null,
+      "outputs": []
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "provenance": [],
+      "include_colab_link": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "name": "python"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

lo-achievement/ai_classroom_suite/IOHelperUtilities.py

@@ -0,0 +1,85 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/helper_utilities.ipynb.
+
+# %% auto 0
+__all__ = ['check_is_colab', 'MultiFileChooser', 'setup_drives']
+
+# %% ../nbs/helper_utilities.ipynb 3
+import ipywidgets as widgets
+from IPython.display import display, clear_output
+from functools import partial
+from ipyfilechooser import FileChooser
+import os
+
+# %% ../nbs/helper_utilities.ipynb 4
+def check_is_colab():
+    """
+    Check if the current environment is Google Colab.
+    """
+    try:
+        import google.colab
+        return True
+    except:
+        return False
+
+# %% ../nbs/helper_utilities.ipynb 7
+class MultiFileChooser:
+    def __init__(self):
+        self.fc = FileChooser('.')
+        self.fc.title = "Use the following file chooser to add each file individually.\n You can remove files by clicking the remove button."
+        self.fc.use_dir_icons = True
+        self.fc.show_only_dirs = False
+        self.selected_files = []
+        
+        self.fc.register_callback(self.file_selected)
+        
+        self.output = widgets.Output()
+        
+    def file_selected(self, chooser):
+        if self.fc.selected is not None and self.fc.selected not in self.selected_files:
+            self.selected_files.append(self.fc.selected)
+            self.update_display()
+    
+    def update_display(self):
+        with self.output:
+            clear_output()
+            for this_file in self.selected_files:
+                remove_button = widgets.Button(description="Remove", tooltip="Remove this file")
+                remove_button.on_click(partial(self.remove_file, file=this_file))
+                display(widgets.HBox([widgets.Label(value=this_file), remove_button]))
+    
+    def remove_file(self, button, this_file):
+        if this_file in self.selected_files:
+            self.selected_files.remove(this_file)
+            self.update_display()
+    
+    def display(self):
+        display(self.fc, self.output)
+    
+    def get_selected_files(self):
+        return self.selected_files
+
+# %% ../nbs/helper_utilities.ipynb 12
+def setup_drives(upload_set):
+
+    upload_set = upload_set.lower()
+    uploaded = None
+
+    # allow them to mount the drive if they chose Google Colab.
+    if upload_set == 'google drive':
+        if check_is_colab():
+            from google.colab import drive
+            drive.mount('/content/drive')
+        else:
+            raise ValueError("It looks like you're not on Google Colab. Google Drive mounting is currently only implemented for Google Colab.")
+
+    # Everything else means that they'll need to use a file chooser (including Google Drive)
+    if check_is_colab():
+        from google.colab import files
+        uploaded = files.upload()
+    else:
+        # Create file chooser and interact
+        mfc = MultiFileChooser()
+        mfc.display()
+        uploaded = mfc.get_selected_files()
+    
+    return uploaded

lo-achievement/ai_classroom_suite/MediaVectorStores.py

@@ -0,0 +1,173 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/media_stores.ipynb.
+
+# %% auto 0
+__all__ = ['rawtext_to_doc_split', 'files_to_text', 'youtube_to_text', 'save_text', 'get_youtube_transcript',
+           'website_to_text_web', 'website_to_text_unstructured', 'get_document_segments', 'create_local_vector_store']
+
+# %% ../nbs/media_stores.ipynb 3
+# import libraries here
+import os
+import itertools
+
+from langchain.embeddings import OpenAIEmbeddings
+
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.document_loaders.unstructured import UnstructuredFileLoader
+from langchain.document_loaders.generic import GenericLoader
+from langchain.document_loaders.parsers import OpenAIWhisperParser
+from langchain.document_loaders.blob_loaders.youtube_audio import YoutubeAudioLoader
+from langchain.document_loaders import WebBaseLoader, UnstructuredURLLoader
+from langchain.docstore.document import Document
+
+from langchain.vectorstores import Chroma
+from langchain.chains import RetrievalQAWithSourcesChain
+
+# %% ../nbs/media_stores.ipynb 8
+def rawtext_to_doc_split(text, chunk_size=1500, chunk_overlap=150):
+  
+  # Quick type checking
+  if not isinstance(text, list):
+    text = [text]
+
+  # Create splitter
+  text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size,
+                                                 chunk_overlap=chunk_overlap,
+                                                 add_start_index = True)
+  
+  #Split into docs segments
+  if isinstance(text[0], Document):
+    doc_segments = text_splitter.split_documents(text)
+  else:
+    doc_segments = text_splitter.split_documents(text_splitter.create_documents(text))
+
+  # Make into one big list
+  doc_segments = list(itertools.chain(*doc_segments)) if isinstance(doc_segments[0], list) else doc_segments
+
+  return doc_segments
+
+# %% ../nbs/media_stores.ipynb 16
+## A single File
+def _file_to_text(single_file, chunk_size = 1000, chunk_overlap=150):
+
+  # Create loader and get segments
+  loader = UnstructuredFileLoader(single_file)
+  doc_segments = loader.load_and_split(RecursiveCharacterTextSplitter(chunk_size=chunk_size,
+                                                                      chunk_overlap=chunk_overlap,
+                                                                      add_start_index=True))
+  return doc_segments
+
+
+## Multiple files
+def files_to_text(files_list, chunk_size=1000, chunk_overlap=150):
+  
+  # Quick type checking
+  if not isinstance(files_list, list):
+    files_list = [files_list]
+
+  # This is currently a fix because the UnstructuredFileLoader expects a list of files yet can't split them correctly yet
+  all_segments = [_file_to_text(single_file, chunk_size=chunk_size, chunk_overlap=chunk_overlap) for single_file in files_list]
+  all_segments = list(itertools.chain(*all_segments)) if isinstance(all_segments[0], list) else all_segments
+
+  return all_segments
+
+# %% ../nbs/media_stores.ipynb 20
+def youtube_to_text(urls, save_dir = "content"):
+  # Transcribe the videos to text
+  # save_dir: directory to save audio files
+
+  if not isinstance(urls, list):
+    urls = [urls]
+  
+  youtube_loader = GenericLoader(YoutubeAudioLoader(urls, save_dir), OpenAIWhisperParser())
+  youtube_docs = youtube_loader.load()
+  
+  return youtube_docs
+
+# %% ../nbs/media_stores.ipynb 24
+def save_text(text, text_name = None):
+  if not text_name:
+    text_name = text[:20]
+  text_path = os.path.join("/content",text_name+".txt")
+  
+  with open(text_path, "x") as f:
+    f.write(text)
+  # Return the location at which the transcript is saved
+  return text_path
+
+# %% ../nbs/media_stores.ipynb 25
+def get_youtube_transcript(yt_url, save_transcript = False, temp_audio_dir = "sample_data"):
+  # Transcribe the videos to text and save to file in /content
+  # save_dir: directory to save audio files
+
+  youtube_docs = youtube_to_text(yt_url, save_dir = temp_audio_dir)
+  
+  # Combine doc
+  combined_docs = [doc.page_content for doc in youtube_docs]
+  combined_text = " ".join(combined_docs)
+  
+  # Save text to file
+  video_path = youtube_docs[0].metadata["source"]
+  youtube_name = os.path.splitext(os.path.basename(video_path))[0]
+
+  save_path = None
+  if save_transcript:
+    save_path = save_text(combined_text, youtube_name)
+  
+  return youtube_docs, save_path
+
+# %% ../nbs/media_stores.ipynb 27
+def website_to_text_web(url, chunk_size = 1500, chunk_overlap=100):
+  
+    # Url can be a single string or list
+    website_loader = WebBaseLoader(url)
+    website_raw = website_loader.load()
+
+    website_data = rawtext_to_doc_split(website_raw, chunk_size = chunk_size, chunk_overlap=chunk_overlap)
+  
+    # Combine doc
+    return website_data
+
+# %% ../nbs/media_stores.ipynb 33
+def website_to_text_unstructured(web_urls, chunk_size = 1500, chunk_overlap=100):
+
+    # Make sure it's a list
+    if not isinstance(web_urls, list):
+        web_urls = [web_urls]
+  
+    # Url can be a single string or list
+    website_loader = UnstructuredURLLoader(web_urls)
+    website_raw = website_loader.load()
+
+    website_data = rawtext_to_doc_split(website_raw, chunk_size = chunk_size, chunk_overlap=chunk_overlap)
+  
+    # Return individual docs or list
+    return website_data
+
+# %% ../nbs/media_stores.ipynb 45
+def get_document_segments(context_info, data_type, chunk_size = 1500, chunk_overlap=100):
+
+    load_fcn = None
+    addtnl_params = {'chunk_size': chunk_size, 'chunk_overlap': chunk_overlap}
+
+    # Define function use to do the loading
+    if data_type == 'text':
+        load_fcn = rawtext_to_doc_split
+    elif data_type == 'web_page':
+        load_fcn = website_to_text_unstructured
+    elif data_type == 'youtube_video':
+        load_fcn = youtube_to_text
+    else:
+        load_fcn = files_to_text
+    
+    # Get the document segments
+    doc_segments = load_fcn(context_info, **addtnl_params)
+
+    return doc_segments
+
+# %% ../nbs/media_stores.ipynb 47
+def create_local_vector_store(document_segments, **retriever_kwargs):
+    embeddings = OpenAIEmbeddings()
+    db = Chroma.from_documents(document_segments, embeddings)
+    retriever = db.as_retriever(**retriever_kwargs)
+    
+    return db, retriever

lo-achievement/ai_classroom_suite/PromptInteractionBase.py

@@ -0,0 +1,191 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/prompt_interaction_base.ipynb.
+
+# %% auto 0
+__all__ = ['SYSTEM_TUTOR_TEMPLATE', 'HUMAN_RESPONSE_TEMPLATE', 'HUMAN_RETRIEVER_RESPONSE_TEMPLATE', 'DEFAULT_ASSESSMENT_MSG',
+           'DEFAULT_LEARNING_OBJS_MSG', 'DEFAULT_CONDENSE_PROMPT_TEMPLATE', 'DEFAULT_QUESTION_PROMPT_TEMPLATE',
+           'DEFAULT_COMBINE_PROMPT_TEMPLATE', 'create_model', 'set_openai_key', 'create_base_tutoring_prompt',
+           'get_tutoring_prompt', 'get_tutoring_answer', 'create_tutor_mdl_chain']
+
+# %% ../nbs/prompt_interaction_base.ipynb 3
+from langchain.chat_models import ChatOpenAI
+from langchain.llms import OpenAI
+
+from langchain import PromptTemplate
+from langchain.prompts import ChatPromptTemplate, PromptTemplate
+from langchain.prompts import SystemMessagePromptTemplate, HumanMessagePromptTemplate
+from langchain.chains import LLMChain, ConversationalRetrievalChain, RetrievalQAWithSourcesChain
+from langchain.chains.base import Chain
+
+from getpass import getpass
+
+import os
+
+# %% ../nbs/prompt_interaction_base.ipynb 5
+def create_model(openai_mdl='gpt-3.5-turbo-16k', temperature=0.1, **chatopenai_kwargs):
+    llm = ChatOpenAI(model_name = openai_mdl, temperature=temperature, **chatopenai_kwargs)
+
+    return llm
+
+# %% ../nbs/prompt_interaction_base.ipynb 6
+def set_openai_key():
+    openai_api_key = getpass()
+    os.environ["OPENAI_API_KEY"] = openai_api_key
+
+    return
+
+# %% ../nbs/prompt_interaction_base.ipynb 10
+# Create system prompt template
+SYSTEM_TUTOR_TEMPLATE = ("You are a world-class tutor helping students to perform better on oral and written exams though interactive experiences. " +
+                         "When assessing and evaluating students, you always ask one question at a time, and wait for the student's response before " +
+                         "providing them with feedback. Asking one question at a time, waiting for the student's response, and then commenting " +
+                         "on the strengths and weaknesses of their responses (when appropriate) is what makes you such a sought-after, world-class tutor.")
+
+# Create a human response template
+HUMAN_RESPONSE_TEMPLATE = ("I'm trying to better understand the text provided below. {assessment_request} The learning objectives to be assessed are: " +
+                           "{learning_objectives}. Although I may request more than one assessment question, you should " +
+                           "only provide ONE question in you initial response. Do not include the answer in your response. " +
+                           "If I get an answer wrong, provide me with an explanation of why it was incorrect, and then give me additional " +
+                           "chances to respond until I get the correct choice. Explain why the correct choice is right. " +
+                           "The text that you will base your questions on is as follows: {context}.")
+
+HUMAN_RETRIEVER_RESPONSE_TEMPLATE = ("I want to master the topics based on the excerpts of the text below. Given the following extracted text from long documents, {assessment_request} The learning objectives to be assessed are: " +
+                           "{learning_objectives}. Although I may request more than one assessment question, you should " +
+                           "only provide ONE question in you initial response. Do not include the answer in your response. " +
+                           "If I get an answer wrong, provide me with an explanation of why it was incorrect, and then give me additional " +
+                           "chances to respond until I get the correct choice. Explain why the correct choice is right. " +
+                           "The extracted text from long documents are as follows: {summaries}.")
+
+def create_base_tutoring_prompt(system_prompt=None, human_prompt=None):
+
+    #setup defaults using defined values
+    if system_prompt == None:
+        system_prompt = PromptTemplate(template = SYSTEM_TUTOR_TEMPLATE,
+                          input_variables = [])
+    
+    if human_prompt==None:
+        human_prompt = PromptTemplate(template = HUMAN_RESPONSE_TEMPLATE,
+                           input_variables=['assessment_request', 'learning_objectives', 'context'])
+
+    # Create prompt messages
+    system_tutor_msg = SystemMessagePromptTemplate(prompt=system_prompt)
+    human_tutor_msg = HumanMessagePromptTemplate(prompt= human_prompt)
+
+    # Create ChatPromptTemplate
+    chat_prompt = ChatPromptTemplate.from_messages([system_tutor_msg, human_tutor_msg])
+
+    return chat_prompt
+
+# %% ../nbs/prompt_interaction_base.ipynb 14
+DEFAULT_ASSESSMENT_MSG = 'Please design a 5 question short answer quiz about the provided text.'
+DEFAULT_LEARNING_OBJS_MSG = 'Identify and comprehend the important topics and underlying messages and connections within the text'
+
+def get_tutoring_prompt(context, chat_template=None, assessment_request = None, learning_objectives = None, **kwargs):
+
+    # set defaults
+    if chat_template is None:
+        chat_template = create_base_tutoring_prompt()
+    else:
+        if not all([prompt_var in chat_template.input_variables
+                    for prompt_var in ['context', 'assessment_request', 'learning_objectives']]):
+            raise KeyError('''It looks like you may have a custom chat_template. Either include context, assessment_request, and learning objectives
+                           as input variables or create your own tutoring prompt.''')
+
+    if assessment_request is None and 'assessment_request':
+        assessment_request = DEFAULT_ASSESSMENT_MSG
+    
+    if learning_objectives is None:
+        learning_objectives = DEFAULT_LEARNING_OBJS_MSG
+    
+    # compose final prompt
+    tutoring_prompt = chat_template.format_prompt(context=context,
+                                                assessment_request = assessment_request,
+                                                learning_objectives = learning_objectives,
+                                                **kwargs)
+    
+    return tutoring_prompt
+
+
+# %% ../nbs/prompt_interaction_base.ipynb 18
+def get_tutoring_answer(context, tutor_mdl, chat_template=None, assessment_request=None, learning_objectives=None, return_dict=False, call_kwargs={}, input_kwargs={}):
+    
+ # Get answer from chat
+    
+    # set defaults
+    if assessment_request is None:
+            assessment_request = DEFAULT_ASSESSMENT_MSG
+    if learning_objectives is None:
+        learning_objectives = DEFAULT_LEARNING_OBJS_MSG
+    
+    common_inputs = {'assessment_request':assessment_request, 'learning_objectives':learning_objectives}
+    
+    # get answer based on interaction type
+    if isinstance(tutor_mdl, ChatOpenAI):
+        human_ask_prompt = get_tutoring_prompt(context, chat_template, assessment_request, learning_objectives)
+        tutor_answer = tutor_mdl(human_ask_prompt.to_messages())
+
+        if not return_dict:
+            final_answer = tutor_answer.content
+    
+    elif isinstance(tutor_mdl, Chain):
+        if isinstance(tutor_mdl, RetrievalQAWithSourcesChain):
+            if 'question' not in input_kwargs.keys():
+                common_inputs['question'] = assessment_request
+            final_inputs = {**common_inputs, **input_kwargs}
+        else:
+            common_inputs['context'] = context
+            final_inputs = {**common_inputs, **input_kwargs}
+         
+        # get answer
+        tutor_answer = tutor_mdl(final_inputs, **call_kwargs)
+        final_answer = tutor_answer
+
+        if not return_dict:
+            final_answer = final_answer['answer']
+    
+    else:
+        raise NotImplementedError(f"tutor_mdl of type {type(tutor_mdl)} is not supported.")
+
+    return final_answer
+
+# %% ../nbs/prompt_interaction_base.ipynb 19
+DEFAULT_CONDENSE_PROMPT_TEMPLATE = ("Given the following conversation and a follow up question, rephrase the follow up question to be a standalone question, " + 
+                                    "in its original language.\n\nChat History:\n{chat_history}\nFollow Up Input: {question}\nStandalone question:")
+
+DEFAULT_QUESTION_PROMPT_TEMPLATE  = ("Use the following portion of a long document to see if any of the text is relevant to creating a response to the question." +
+                                     "\nReturn any relevant text verbatim.\n{context}\nQuestion: {question}\nRelevant text, if any:")
+
+DEFAULT_COMBINE_PROMPT_TEMPLATE = ("Given the following extracted parts of a long document and the given prompt, create a final answer with references ('SOURCES'). "+
+                                   "If you don't have a response, just say that you are unable to come up with a response. "+
+                                   "\nSOURCES:\n\nQUESTION: {question}\n=========\n{summaries}\n=========\nFINAL ANSWER:'")
+
+def create_tutor_mdl_chain(kind='llm', mdl=None, prompt_template = None, **kwargs):
+    
+    #Validate parameters
+    if mdl is None:
+        mdl = create_model()
+    kind = kind.lower()
+    
+    #Create model chain
+    if kind == 'llm':
+        if prompt_template is None:
+            prompt_template = create_base_tutoring_prompt()
+        mdl_chain = LLMChain(llm=mdl, prompt=prompt_template, **kwargs)
+    elif kind == 'conversational':
+        if prompt_template is None:
+            prompt_template = PromptTemplate.from_template(DEFAULT_CONDENSE_PROMPT_TEMPLATE)
+        mdl_chain = ConversationalRetrieverChain.from_llm(mdl, condense_question_prompt = prompt_template, **kwargs)
+    elif kind == 'retrieval_qa':
+        if prompt_template is None:
+
+            #Create custom human prompt to take in summaries
+            human_prompt = PromptTemplate(template = HUMAN_RETRIEVER_RESPONSE_TEMPLATE,
+                           input_variables=['assessment_request', 'learning_objectives', 'summaries'])
+            prompt_template = create_base_tutoring_prompt(human_prompt=human_prompt)
+            
+        #Create the combination prompt and model
+        question_template = PromptTemplate.from_template(DEFAULT_QUESTION_PROMPT_TEMPLATE)
+        mdl_chain = RetrievalQAWithSourcesChain.from_llm(llm=mdl, question_prompt=question_template, combine_prompt = prompt_template, **kwargs)
+    else:
+        raise NotImplementedError(f"Model kind {kind} not implemented")
+    
+    return mdl_chain

lo-achievement/ai_classroom_suite/SelfStudyPrompts.py

@@ -0,0 +1,75 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/self_study_prompts.ipynb.
+
+# %% auto 0
+__all__ = ['MC_QUIZ_DEFAULT', 'SHORT_ANSWER_DEFAULT', 'FILL_BLANK_DEFAULT', 'SEQUENCING_DEFAULT', 'RELATIONSHIP_DEFAULT',
+           'CONCEPTS_DEFAULT', 'REAL_WORLD_EXAMPLE_DEFAULT', 'RANDOMIZED_QUESTIONS_DEFAULT', 'SELF_STUDY_PROMPT_NAMES',
+           'SELF_STUDY_DEFAULTS', 'list_all_self_study_prompt_keys', 'list_all_self_study_prompts',
+           'list_default_self_prompt_varnames', 'print_all_self_study_prompts']
+
+# %% ../nbs/self_study_prompts.ipynb 4
+# used for pretty display
+import pandas as pd
+
+# %% ../nbs/self_study_prompts.ipynb 5
+MC_QUIZ_DEFAULT = "Please design a 5 question multiple choice quiz about the provided text."
+
+SHORT_ANSWER_DEFAULT = ("Please design a 5 question short answer quiz about the provided text. "
+                        "The question types should be short answer. Expect the correct answers to be a few sentences long.")
+
+FILL_BLANK_DEFAULT = """Create a 5 question fill in the blank quiz referencing parts of the provided text.
+The "blank" part of the question should appear as "________". The answers should reflect what word(s) should go in the blank an accurate statement.
+An example is as follows: "The author of the book is ______." The question should be a statement.
+"""
+
+SEQUENCING_DEFAULT = """Create a 5 question questionnaire that will ask me to recall the steps or sequence of events
+in the provided text."""
+
+RELATIONSHIP_DEFAULT = ("Create a 5 question quiz for the student that asks the student to identify relationships between"
+                        "topics or concepts that are important to understanding this text.")
+
+CONCEPTS_DEFAULT = """ Design a 5 question quiz that asks me about definitions or concepts of importance in the provided text."""
+
+REAL_WORLD_EXAMPLE_DEFAULT = """Demonstrate how the provided context can be applied to solve a real world problem.
+Ask me questions about how the demonstration you provided relates to solving a real world problem."""
+
+RANDOMIZED_QUESTIONS_DEFAULT = """Generate a high-quality assessment consisting of 5 varied questions,
+each of different types (open-ended, multiple choice, short answer, analogies, etc.)"""
+
+SELF_STUDY_PROMPT_NAMES = ['MC_QUIZ_DEFAULT',
+'SHORT_ANSWER_DEFAULT',
+'FILL_BLANK_DEFAULT',
+'SEQUENCING_DEFAULT',
+'RELATIONSHIP_DEFAULT',
+'CONCEPTS_DEFAULT',
+'REAL_WORLD_EXAMPLE_DEFAULT',
+'RANDOMIZED_QUESTIONS_DEFAULT']
+
+# %% ../nbs/self_study_prompts.ipynb 7
+# Define self study dictionary for lookup
+SELF_STUDY_DEFAULTS = {'mc': MC_QUIZ_DEFAULT,
+'short_answer': SHORT_ANSWER_DEFAULT,
+'fill_blank': FILL_BLANK_DEFAULT,
+'sequencing': SEQUENCING_DEFAULT,
+'relationships': RELATIONSHIP_DEFAULT,
+'concepts': CONCEPTS_DEFAULT,
+'real_world_example': REAL_WORLD_EXAMPLE_DEFAULT,
+'randomized_questions': RANDOMIZED_QUESTIONS_DEFAULT
+} 
+
+# Return list of all self study prompts
+def list_all_self_study_prompt_keys():
+    return list(SELF_STUDY_DEFAULTS.keys())
+
+def list_all_self_study_prompts():
+    return list(SELF_STUDY_DEFAULTS.values())
+    
+# Return list of all self study variable names
+def list_default_self_prompt_varnames():
+    return SELF_STUDY_PROMPT_NAMES
+
+# Print as a table
+def print_all_self_study_prompts():
+    with pd.option_context('max_colwidth', None):
+        display(pd.DataFrame({'SELF_STUDY_DEFAULTS key': list(SELF_STUDY_DEFAULTS.keys()),
+        'Prompt': list(SELF_STUDY_DEFAULTS.values())}))
+

lo-achievement/ai_classroom_suite/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.0.1"

lo-achievement/ai_classroom_suite/_modidx.py

@@ -0,0 +1,93 @@
+# Autogenerated by nbdev
+
+d = { 'settings': { 'branch': 'main',
+                'doc_baseurl': '/lo-achievement',
+                'doc_host': 'https://vanderbilt-data-science.github.io',
+                'git_url': 'https://github.com/vanderbilt-data-science/lo-achievement',
+                'lib_path': 'ai_classroom_suite'},
+  'syms': { 'ai_classroom_suite.IOHelperUtilities': { 'ai_classroom_suite.IOHelperUtilities.MultiFileChooser': ( 'helper_utilities.html#multifilechooser',
+                                                                                                                 'ai_classroom_suite/IOHelperUtilities.py'),
+                                                      'ai_classroom_suite.IOHelperUtilities.MultiFileChooser.__init__': ( 'helper_utilities.html#multifilechooser.__init__',
+                                                                                                                          'ai_classroom_suite/IOHelperUtilities.py'),
+                                                      'ai_classroom_suite.IOHelperUtilities.MultiFileChooser.display': ( 'helper_utilities.html#multifilechooser.display',
+                                                                                                                         'ai_classroom_suite/IOHelperUtilities.py'),
+                                                      'ai_classroom_suite.IOHelperUtilities.MultiFileChooser.file_selected': ( 'helper_utilities.html#multifilechooser.file_selected',
+                                                                                                                               'ai_classroom_suite/IOHelperUtilities.py'),
+                                                      'ai_classroom_suite.IOHelperUtilities.MultiFileChooser.get_selected_files': ( 'helper_utilities.html#multifilechooser.get_selected_files',
+                                                                                                                                    'ai_classroom_suite/IOHelperUtilities.py'),
+                                                      'ai_classroom_suite.IOHelperUtilities.MultiFileChooser.remove_file': ( 'helper_utilities.html#multifilechooser.remove_file',
+                                                                                                                             'ai_classroom_suite/IOHelperUtilities.py'),
+                                                      'ai_classroom_suite.IOHelperUtilities.MultiFileChooser.update_display': ( 'helper_utilities.html#multifilechooser.update_display',
+                                                                                                                                'ai_classroom_suite/IOHelperUtilities.py'),
+                                                      'ai_classroom_suite.IOHelperUtilities.check_is_colab': ( 'helper_utilities.html#check_is_colab',
+                                                                                                               'ai_classroom_suite/IOHelperUtilities.py'),
+                                                      'ai_classroom_suite.IOHelperUtilities.setup_drives': ( 'helper_utilities.html#setup_drives',
+                                                                                                             'ai_classroom_suite/IOHelperUtilities.py')},
+            'ai_classroom_suite.MediaVectorStores': { 'ai_classroom_suite.MediaVectorStores._file_to_text': ( 'media_stores.html#_file_to_text',
+                                                                                                              'ai_classroom_suite/MediaVectorStores.py'),
+                                                      'ai_classroom_suite.MediaVectorStores.create_local_vector_store': ( 'media_stores.html#create_local_vector_store',
+                                                                                                                          'ai_classroom_suite/MediaVectorStores.py'),
+                                                      'ai_classroom_suite.MediaVectorStores.files_to_text': ( 'media_stores.html#files_to_text',
+                                                                                                              'ai_classroom_suite/MediaVectorStores.py'),
+                                                      'ai_classroom_suite.MediaVectorStores.get_document_segments': ( 'media_stores.html#get_document_segments',
+                                                                                                                      'ai_classroom_suite/MediaVectorStores.py'),
+                                                      'ai_classroom_suite.MediaVectorStores.get_youtube_transcript': ( 'media_stores.html#get_youtube_transcript',
+                                                                                                                       'ai_classroom_suite/MediaVectorStores.py'),
+                                                      'ai_classroom_suite.MediaVectorStores.rawtext_to_doc_split': ( 'media_stores.html#rawtext_to_doc_split',
+                                                                                                                     'ai_classroom_suite/MediaVectorStores.py'),
+                                                      'ai_classroom_suite.MediaVectorStores.save_text': ( 'media_stores.html#save_text',
+                                                                                                          'ai_classroom_suite/MediaVectorStores.py'),
+                                                      'ai_classroom_suite.MediaVectorStores.website_to_text_unstructured': ( 'media_stores.html#website_to_text_unstructured',
+                                                                                                                             'ai_classroom_suite/MediaVectorStores.py'),
+                                                      'ai_classroom_suite.MediaVectorStores.website_to_text_web': ( 'media_stores.html#website_to_text_web',
+                                                                                                                    'ai_classroom_suite/MediaVectorStores.py'),
+                                                      'ai_classroom_suite.MediaVectorStores.youtube_to_text': ( 'media_stores.html#youtube_to_text',
+                                                                                                                'ai_classroom_suite/MediaVectorStores.py')},
+            'ai_classroom_suite.PromptInteractionBase': { 'ai_classroom_suite.PromptInteractionBase.create_base_tutoring_prompt': ( 'prompt_interaction_base.html#create_base_tutoring_prompt',
+                                                                                                                                    'ai_classroom_suite/PromptInteractionBase.py'),
+                                                          'ai_classroom_suite.PromptInteractionBase.create_model': ( 'prompt_interaction_base.html#create_model',
+                                                                                                                     'ai_classroom_suite/PromptInteractionBase.py'),
+                                                          'ai_classroom_suite.PromptInteractionBase.create_tutor_mdl_chain': ( 'prompt_interaction_base.html#create_tutor_mdl_chain',
+                                                                                                                               'ai_classroom_suite/PromptInteractionBase.py'),
+                                                          'ai_classroom_suite.PromptInteractionBase.get_tutoring_answer': ( 'prompt_interaction_base.html#get_tutoring_answer',
+                                                                                                                            'ai_classroom_suite/PromptInteractionBase.py'),
+                                                          'ai_classroom_suite.PromptInteractionBase.get_tutoring_prompt': ( 'prompt_interaction_base.html#get_tutoring_prompt',
+                                                                                                                            'ai_classroom_suite/PromptInteractionBase.py'),
+                                                          'ai_classroom_suite.PromptInteractionBase.set_openai_key': ( 'prompt_interaction_base.html#set_openai_key',
+                                                                                                                       'ai_classroom_suite/PromptInteractionBase.py')},
+            'ai_classroom_suite.SelfStudyPrompts': { 'ai_classroom_suite.SelfStudyPrompts.list_all_self_study_prompt_keys': ( 'self_study_prompts.html#list_all_self_study_prompt_keys',
+                                                                                                                              'ai_classroom_suite/SelfStudyPrompts.py'),
+                                                     'ai_classroom_suite.SelfStudyPrompts.list_all_self_study_prompts': ( 'self_study_prompts.html#list_all_self_study_prompts',
+                                                                                                                          'ai_classroom_suite/SelfStudyPrompts.py'),
+                                                     'ai_classroom_suite.SelfStudyPrompts.list_default_self_prompt_varnames': ( 'self_study_prompts.html#list_default_self_prompt_varnames',
+                                                                                                                                'ai_classroom_suite/SelfStudyPrompts.py'),
+                                                     'ai_classroom_suite.SelfStudyPrompts.print_all_self_study_prompts': ( 'self_study_prompts.html#print_all_self_study_prompts',
+                                                                                                                           'ai_classroom_suite/SelfStudyPrompts.py')},
+            'ai_classroom_suite.self_study_app': { 'ai_classroom_suite.self_study_app.SlightlyDelusionalTutor': ( 'gradio_application.html#slightlydelusionaltutor',
+                                                                                                                  'ai_classroom_suite/self_study_app.py'),
+                                                   'ai_classroom_suite.self_study_app.SlightlyDelusionalTutor.__init__': ( 'gradio_application.html#slightlydelusionaltutor.__init__',
+                                                                                                                           'ai_classroom_suite/self_study_app.py'),
+                                                   'ai_classroom_suite.self_study_app.SlightlyDelusionalTutor.add_user_message': ( 'gradio_application.html#slightlydelusionaltutor.add_user_message',
+                                                                                                                                   'ai_classroom_suite/self_study_app.py'),
+                                                   'ai_classroom_suite.self_study_app.SlightlyDelusionalTutor.forget_conversation': ( 'gradio_application.html#slightlydelusionaltutor.forget_conversation',
+                                                                                                                                      'ai_classroom_suite/self_study_app.py'),
+                                                   'ai_classroom_suite.self_study_app.SlightlyDelusionalTutor.get_sources_memory': ( 'gradio_application.html#slightlydelusionaltutor.get_sources_memory',
+                                                                                                                                     'ai_classroom_suite/self_study_app.py'),
+                                                   'ai_classroom_suite.self_study_app.SlightlyDelusionalTutor.get_tutor_reply': ( 'gradio_application.html#slightlydelusionaltutor.get_tutor_reply',
+                                                                                                                                  'ai_classroom_suite/self_study_app.py'),
+                                                   'ai_classroom_suite.self_study_app.SlightlyDelusionalTutor.initialize_llm': ( 'gradio_application.html#slightlydelusionaltutor.initialize_llm',
+                                                                                                                                 'ai_classroom_suite/self_study_app.py'),
+                                                   'ai_classroom_suite.self_study_app.add_user_message': ( 'gradio_application.html#add_user_message',
+                                                                                                           'ai_classroom_suite/self_study_app.py'),
+                                                   'ai_classroom_suite.self_study_app.create_reference_store': ( 'gradio_application.html#create_reference_store',
+                                                                                                                 'ai_classroom_suite/self_study_app.py'),
+                                                   'ai_classroom_suite.self_study_app.disable_until_done': ( 'gradio_application.html#disable_until_done',
+                                                                                                             'ai_classroom_suite/self_study_app.py'),
+                                                   'ai_classroom_suite.self_study_app.embed_key': ( 'gradio_application.html#embed_key',
+                                                                                                    'ai_classroom_suite/self_study_app.py'),
+                                                   'ai_classroom_suite.self_study_app.get_tutor_reply': ( 'gradio_application.html#get_tutor_reply',
+                                                                                                          'ai_classroom_suite/self_study_app.py'),
+                                                   'ai_classroom_suite.self_study_app.prompt_select': ( 'gradio_application.html#prompt_select',
+                                                                                                        'ai_classroom_suite/self_study_app.py'),
+                                                   'ai_classroom_suite.self_study_app.save_chatbot_dialogue': ( 'gradio_application.html#save_chatbot_dialogue',
+                                                                                                                'ai_classroom_suite/self_study_app.py')}}}

lo-achievement/ai_classroom_suite/self_study_app.py

@@ -0,0 +1,358 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/gradio_application.ipynb.
+
+# %% auto 0
+__all__ = ['save_pdf', 'save_json', 'save_txt', 'save_csv', 'num_sources', 'css', 'save_chatbot_dialogue',
+           'SlightlyDelusionalTutor', 'embed_key', 'create_reference_store', 'prompt_select', 'add_user_message',
+           'get_tutor_reply', 'disable_until_done']
+
+# %% ../nbs/gradio_application.ipynb 9
+import gradio as gr
+from functools import partial
+import pandas as pd
+import os
+
+from .PromptInteractionBase import *
+from .IOHelperUtilities import *
+from .SelfStudyPrompts import *
+from .MediaVectorStores import *
+
+# %% ../nbs/gradio_application.ipynb 13
+def save_chatbot_dialogue(chat_tutor, save_type):
+
+    formatted_convo = pd.DataFrame(chat_tutor.conversation_memory, columns=['user', 'chatbot'])
+
+    output_fname = f'tutoring_conversation.{save_type}'
+
+    if save_type == 'csv':
+        formatted_convo.to_csv(output_fname, index=False)
+    elif save_type == 'json':
+        formatted_convo.to_json(output_fname, orient='records')
+    elif save_type == 'txt':
+        temp = formatted_convo.apply(lambda x: 'User: {0}\nAI: {1}'.format(x[0], x[1]), axis=1)
+        temp = '\n\n'.join(temp.tolist())
+        with open(output_fname, 'w') as f:
+            f.write(temp)
+    else:
+        gr.update(value=None, visible=False)
+    
+    return gr.update(value=output_fname, visible=True)
+
+save_pdf = partial(save_chatbot_dialogue, save_type='pdf')
+save_json = partial(save_chatbot_dialogue, save_type='json')
+save_txt = partial(save_chatbot_dialogue, save_type='txt')
+save_csv = partial(save_chatbot_dialogue, save_type='csv')
+
+
+# %% ../nbs/gradio_application.ipynb 16
+class SlightlyDelusionalTutor:
+    # create basic initialization function
+    def __init__(self, model_name = None):
+
+        # create default model name
+        if model_name is None:
+            self.model_name = 'gpt-3.5-turbo-16k'
+
+        self.chat_llm = None
+        self.tutor_chain = None
+        self.vector_store = None
+        self.vs_retriever = None
+        self.conversation_memory = []
+        self.sources_memory = []
+        self.flattened_conversation = ''
+        self.api_key_valid = False
+        self.learning_objectives = None
+        self.openai_auth = ''
+    
+    def initialize_llm(self):
+
+        if self.openai_auth:
+            try:
+                self.chat_llm = create_model(self.model_name, openai_api_key = self.openai_auth)
+                self.api_key_valid = True
+            except Exception as e:
+                print(e)
+                self.api_key_valid = False
+        else:
+            print("Please provide an OpenAI API key and press Enter.")
+
+    def add_user_message(self, user_message):
+        self.conversation_memory.append([user_message, None])
+        self.flattened_conversation = self.flattened_conversation + '\n\n' + 'User: ' + user_message
+    
+    def get_tutor_reply(self, **input_kwargs):
+
+        if not self.conversation_memory:
+            return "Please type something to start the conversation."
+        
+        # we want to have a different vector comparison for reference lookup after the topic is first used
+        if len(self.conversation_memory) > 1:
+            if 'question' in input_kwargs.keys():
+                if input_kwargs['question']:
+                    input_kwargs['question'] = self.conversation_memory[-1][0] + ' keeping in mind I want to learn about ' + input_kwargs['question']
+                else:
+                    input_kwargs['question'] = self.conversation_memory[-1][0]
+
+        # get tutor message
+        tutor_message = get_tutoring_answer(None,
+                                     self.tutor_chain,
+                                     assessment_request = self.flattened_conversation + 'First, please provide your feedback on my previous answer if I was answering a question, otherwise, respond appropriately to my statement. Then, help me with the following:' + self.conversation_memory[-1][0],
+                                     learning_objectives = self.learning_objectives,
+                                     return_dict=True,
+                                     **input_kwargs)
+
+        # add tutor message to conversation memory
+        self.conversation_memory[-1][1] = tutor_message['answer']
+        self.flattened_conversation = self.flattened_conversation + '\nAI: ' + tutor_message['answer']
+        self.sources_memory.append(tutor_message['source_documents'])
+        #print(self.flattened_conversation, '\n\n')
+        print(tutor_message['source_documents'])
+    
+    def get_sources_memory(self):
+        # retrieve last source
+        last_sources = self.sources_memory[-1]
+
+        # get page_content keyword from last_sources
+        doc_contents = ['Source ' + str(ind+1) + '\n"' + doc.page_content + '"\n\n' for ind, doc in enumerate(last_sources)]
+        doc_contents = ''.join(doc_contents)
+
+        return doc_contents
+    
+    def forget_conversation(self):
+        self.conversation_memory = []
+        self.sources_memory = []
+        self.flattened_conversation = ''
+
+# %% ../nbs/gradio_application.ipynb 18
+def embed_key(openai_api_key, chat_tutor):
+  if not openai_api_key:
+    return chat_tutor
+  
+  # Otherwise, update key
+  os.environ["OPENAI_API_KEY"] = openai_api_key
+  
+  #update tutor
+  chat_tutor.openai_auth = openai_api_key
+
+  if not chat_tutor.api_key_valid:
+    chat_tutor.initialize_llm()
+
+  return chat_tutor
+
+# %% ../nbs/gradio_application.ipynb 20
+def create_reference_store(chat_tutor, vs_button, text_cp, upload_files, reference_vs, openai_auth, learning_objs):
+
+    text_segs = []
+    upload_segs = []
+    
+    if reference_vs:
+        raise NotImplementedError("Reference Vector Stores are not yet implemented")
+    
+    if text_cp.strip():
+        text_segs = get_document_segments(text_cp, 'text', chunk_size=700, chunk_overlap=100)
+        [doc.metadata.update({'source':'text box'}) for doc in text_segs];
+    
+    if upload_files:
+        print(upload_files)
+        upload_fnames = [f.name for f in upload_files]
+        upload_segs = get_document_segments(upload_fnames, 'file', chunk_size=700, chunk_overlap=100)
+    
+    # get the full list of everything
+    all_segs = text_segs + upload_segs
+    print(all_segs)
+
+    # create the vector store and update tutor
+    vs_db, vs_retriever = create_local_vector_store(all_segs, search_kwargs={"k": 2})
+    chat_tutor.vector_store = vs_db
+    chat_tutor.vs_retriever = vs_retriever
+
+    # create the tutor chain
+    if not chat_tutor.api_key_valid or not chat_tutor.openai_auth:
+        chat_tutor = embed_key(openai_auth, chat_tutor)
+    qa_chain = create_tutor_mdl_chain(kind="retrieval_qa", mdl=chat_tutor.chat_llm, retriever = chat_tutor.vs_retriever, return_source_documents=True)
+    chat_tutor.tutor_chain = qa_chain
+
+    # store learning objectives
+    chat_tutor.learning_objectives = learning_objs
+
+    # return the story
+    return chat_tutor, gr.update(interactive=True, value='Tutor Initialized!')
+
+# %% ../nbs/gradio_application.ipynb 22
+### Gradio Called Functions ###
+
+def prompt_select(selection, number, length):
+  if selection == "Random":
+    prompt = f"Please design a {number} question quiz based on the context provided and the inputted learning objectives (if applicable). The types of questions should be randomized (including multiple choice, short answer, true/false, short answer, etc.).  Provide one question at a time, and wait for my response before providing me with feedback. Again, while the quiz may ask for multiple questions, you should only provide 1 question in you initial response. Do not include the answer in your response. If I get an answer wrong, provide me with an explanation of why it was incorrect, and then give me additional chances to respond until I get the correct choice. Explain why the correct choice is right."
+  elif selection == "Fill in the Blank":
+    prompt = f"Create a {number} question fill in the blank quiz refrencing the context provided. The quiz should reflect the learning objectives (if inputted). The 'blank' part of the question should appear as '________'. The answers should reflect what word(s) should go in the blank an accurate statement. An example is the follow: 'The author of the article is ______.' The question should be a statement. Provide one question at a time, and wait for my response before providing me with feedback. Again, while the quiz may ask for multiple questions, you should only provide ONE question in you initial response. Do not include the answer in your response. If I get an answer wrong, provide me with an explanation of why it was incorrect,and then give me additional chances to respond until I get the correct choice. Explain why the correct choice is right."
+  elif selection == "Short Answer":
+    prompt = f"Please design a {number} question quiz about which reflects the learning objectives (if inputted).  The questions should be short answer. Expect the correct answers to be {length} sentences long. Provide one question at a time, and wait for my response before providing me with feedback. Again, while the quiz may ask for multiple questions, you should only provide ONE question in you initial response. Do not include the answer in your response. If I get an answer wrong, provide me with an explanation of why it was incorrect, and then give me additional chances to respond until I get the correct choice. Explain why the correct answer is right."
+  else:
+    prompt = f"Please design a {number} question {selection.lower()} quiz based on the context provided and the inputted learning objectives (if applicable). Provide one question at a time, and wait for my response before providing me with feedback. Again, while the quiz may ask for multiple questions, you should only provide 1 question in you initial response. Do not include the answer in your response. If I get an answer wrong, provide me with an explanation of why it was incorrect, and then give me additional chances to respond until I get the correct choice. Explain why the correct choice is right."
+  return prompt, prompt
+    
+
+# %% ../nbs/gradio_application.ipynb 24
+### Chatbot Functions ###
+
+def add_user_message(user_message, chat_tutor):
+  """Display user message and update chat history to include it.
+  Also disables user text input until bot is finished (call to reenable_chat())
+  See https://gradio.app/creating-a-chatbot/"""
+  chat_tutor.add_user_message(user_message)
+  return gr.update(value="", interactive=False), chat_tutor.conversation_memory, chat_tutor
+
+def get_tutor_reply(learning_topic, chat_tutor):
+  chat_tutor.get_tutor_reply(input_kwargs={'question':learning_topic})
+  return gr.update(value="", interactive=True), gr.update(visible=True, value=chat_tutor.get_sources_memory()), chat_tutor.conversation_memory, chat_tutor
+
+num_sources = 2
+
+# %% ../nbs/gradio_application.ipynb 25
+def disable_until_done(obj_in):
+    return gr.update(interactive=False)
+
+# %% ../nbs/gradio_application.ipynb 27
+# See https://gradio.app/custom-CSS-and-JS/
+css="""
+#sources-container {
+  overflow: scroll !important; /* Needs to override default formatting */
+  /*max-height: 20em; */ /* Arbitrary value */
+}
+#sources-container > div { padding-bottom: 1em !important; /* Arbitrary value */ }
+.short-height > * > * { min-height: 0 !important; }
+.translucent { opacity: 0.5; }
+.textbox_label { padding-bottom: .5em; }
+"""
+#srcs = [] # Reset sources (db and qa are kept the same for ease of testing)
+
+with gr.Blocks(css=css, analytics_enabled=False) as demo:
+
+  #initialize tutor (with state)
+  study_tutor = gr.State(SlightlyDelusionalTutor())
+
+  # Title
+  gr.Markdown("# Studying with a Slightly Delusional Tutor")
+
+  # API Authentication functionality
+  with gr.Box():
+    gr.Markdown("### OpenAI API Key ")
+    gr.HTML("""<span>Embed your OpenAI API key below; if you haven't created one already, visit
+      <a href="https://platform.openai.com/account/api-keys">platform.openai.com/account/api-keys</a>
+    to sign up for an account and get your personal API key</span>""",
+            elem_classes="textbox_label")
+    api_input = gr.Textbox(show_label=False, type="password", container=False, autofocus=True,
+                      placeholder="●●●●●●●●●●●●●●●●●", value='')
+    api_input.submit(fn=embed_key, inputs=[api_input, study_tutor], outputs=study_tutor)
+    api_input.blur(fn=embed_key, inputs=[api_input, study_tutor], outputs=study_tutor)
+
+  # Reference document functionality (building vector stores)
+  with gr.Box():
+    gr.Markdown("### Add Reference Documents")
+    # TODO Add entry for path to vector store (should be disabled for now)
+    with gr.Row(equal_height=True):
+      text_input = gr.TextArea(label='Copy and paste your text below',
+                                lines=2)
+
+      file_input = gr.Files(label="Load a .txt or .pdf file",
+                            file_types=['.pdf', '.txt'], type="file",
+                            elem_classes="short-height")
+
+      instructor_input = gr.TextArea(label='Enter vector store URL, if given by instructor (WIP)', value='',
+                                lines=2, interactive=False, elem_classes="translucent")
+    
+    # Adding the learning objectives
+    with gr.Box():
+      gr.Markdown("### Optional: Enter Your Learning Objectives")
+      learning_objectives = gr.Textbox(label='If provided by your instructor, please input your learning objectives for this session', value='')
+    
+    # Adding the button to submit all of the settings and create the Chat Tutor Chain.
+    with gr.Row():
+      vs_build_button = gr.Button(value = 'Start Studying with Your Tutor!', scale=1)
+      vs_build_button.click(disable_until_done, vs_build_button, vs_build_button) \
+        .then(create_reference_store, [study_tutor, vs_build_button, text_input, file_input, instructor_input, api_input, learning_objectives],
+                            [study_tutor, vs_build_button])
+
+  
+
+  # Premade question prompts
+  with gr.Box():
+    gr.Markdown("""
+    ## Generate a Premade Prompt
+    Select your type and number of desired questions. Click "Generate Prompt" to get your premade prompt,
+    and then "Insert Prompt into Chat" to copy the text into the chat interface below. \
+    You can also copy the prompt using the icon in the upper right corner and paste directly into the input box when interacting with the model.
+    """)
+    with gr.Row():
+      with gr.Column():
+        question_type = gr.Dropdown(["Multiple Choice", "True or False", "Short Answer", "Fill in the Blank", "Random"], label="Question Type")
+        number_of_questions = gr.Textbox(label="Enter desired number of questions")
+        sa_desired_length = gr.Dropdown(["1-2", "3-4", "5-6", "6 or more"], label = "For short answer questions only, choose the desired sentence length for answers. The default value is 1-2 sentences.")
+      with gr.Column():
+        prompt_button = gr.Button("Generate Prompt")
+        premade_prompt_output = gr.Textbox(label="Generated prompt (save or copy)", show_copy_button=True)
+
+
+  # Chatbot interface
+  gr.Markdown("## Chat with the Model")
+  topic_input = gr.Textbox(label="What topic or concept are you trying to learn more about?")
+  with gr.Row(equal_height=True):
+    with gr.Column(scale=2):
+      chatbot = gr.Chatbot()
+      with gr.Row():
+        user_chat_input = gr.Textbox(label="User input", scale=9)
+        user_chat_submit = gr.Button("Ask/answer model", scale=1)
+
+    # sources
+    with gr.Box(elem_id="sources-container", scale=1):
+      # TODO: Display document sources in a nicer format?
+      gr.HTML(value="<h3 id='sources'>Referenced Sources</h3>")
+      sources_output = gr.Textbox(value='', interactive=False, visible=False, show_label=False)
+      #sources_output = []
+      #for i in range(num_sources):
+      #  source_elem = gr.HTML(visible=False)
+      #  sources_output.append(source_elem)
+
+  #define the behavior of prompt button later since it depends on user_chat_input
+  prompt_button.click(prompt_select,
+                      inputs=[question_type, number_of_questions, sa_desired_length],
+                      outputs=[premade_prompt_output, user_chat_input])
+
+  # Display input and output in three-ish parts
+  # (using asynchronous functions):
+  # First show user input, then show model output when complete
+  # Then wait until the bot provides response and return the result
+  # Finally, allow the user to ask a new question by reenabling input
+  async_response = user_chat_submit.click(add_user_message,
+            [user_chat_input, study_tutor],
+            [user_chat_input, chatbot, study_tutor], queue=False) \
+  .then(get_tutor_reply, [topic_input, study_tutor], [user_chat_input, sources_output, chatbot, study_tutor], queue=True)
+
+  async_response_b = user_chat_input.submit(add_user_message,
+            [user_chat_input, study_tutor],
+            [user_chat_input, chatbot, study_tutor], queue=False) \
+  .then(get_tutor_reply, [topic_input, study_tutor], [user_chat_input, sources_output, chatbot, study_tutor], queue=True)
+
+  with gr.Blocks():
+    gr.Markdown("""
+    ## Export Your Chat History
+    Export your chat history as a .json, PDF file, .txt, or .csv file
+    """)
+    with gr.Row():
+      export_dialogue_button_json = gr.Button("JSON")
+      export_dialogue_button_pdf = gr.Button("PDF")
+      export_dialogue_button_txt = gr.Button("TXT")
+      export_dialogue_button_csv = gr.Button("CSV")
+    
+    file_download = gr.Files(label="Download here",
+                            file_types=['.pdf', '.txt', '.csv', '.json'], type="file", visible=False)
+    
+    export_dialogue_button_json.click(save_json, study_tutor, file_download, show_progress=True)
+    export_dialogue_button_pdf.click(save_pdf, study_tutor, file_download, show_progress=True)
+    export_dialogue_button_txt.click(save_txt, study_tutor, file_download, show_progress=True)
+    export_dialogue_button_csv.click(save_csv, study_tutor, file_download, show_progress=True)
+
+demo.queue()
+demo.launch(debug=True)
+#demo.launch()
+#gr.close_all()

lo-achievement/grading_from_json.ipynb

@@ -0,0 +1,606 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "colab_type": "text",
+    "id": "view-in-github"
+   },
+   "source": [
+    "<a href=\"https://colab.research.google.com/github/vanderbilt-data-science/lo-achievement/blob/main/grading_from_json.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "id": "kfO7rE64ZTI_"
+   },
+   "outputs": [],
+   "source": [
+    "!pip install openai"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "id": "f26sZpe-MCCj"
+   },
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "import openai\n",
+    "import os\n",
+    "import pandas as pd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 614
+    },
+    "id": "BVTr_mR0XIJI",
+    "outputId": "897e41a0-d5e1-4b5f-d254-0a6e0f6aa3fa"
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <div id=\"df-e24b7014-4d98-4fc5-9ff1-07fa5c26ba5e\">\n",
+       "    <div class=\"colab-df-container\">\n",
+       "      <div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>timestamp</th>\n",
+       "      <th>author</th>\n",
+       "      <th>message</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>2023-06-07 08:16:00+00:00</td>\n",
+       "      <td>assistant</td>\n",
+       "      <td>Question 1:\\nWhich of the following statements...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>2023-06-07 08:16:30+00:00</td>\n",
+       "      <td>user</td>\n",
+       "      <td>C</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>2023-06-07 08:17:00+00:00</td>\n",
+       "      <td>assistant</td>\n",
+       "      <td>Correct! Option C is the correct answer...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>2023-06-07 08:17:30+00:00</td>\n",
+       "      <td>assistant</td>\n",
+       "      <td>Question 2:\\nWhich of the following expenses a...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>2023-06-07 08:18:00+00:00</td>\n",
+       "      <td>user</td>\n",
+       "      <td>A</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>2023-06-07 08:18:30+00:00</td>\n",
+       "      <td>assistant</td>\n",
+       "      <td>I'm sorry, but your answer is incorrect...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6</th>\n",
+       "      <td>2023-06-07 08:19:00+00:00</td>\n",
+       "      <td>assistant</td>\n",
+       "      <td>Question 2 (Revised):\\nWhich of the following ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>7</th>\n",
+       "      <td>2023-06-07 08:19:30+00:00</td>\n",
+       "      <td>user</td>\n",
+       "      <td>D</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8</th>\n",
+       "      <td>2023-06-07 08:20:00+00:00</td>\n",
+       "      <td>assistant</td>\n",
+       "      <td>Correct! Option D is the correct answer...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9</th>\n",
+       "      <td>2023-06-07 08:20:30+00:00</td>\n",
+       "      <td>assistant</td>\n",
+       "      <td>Question 3:\\nWhat is the purpose of capitalizi...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>10</th>\n",
+       "      <td>2023-06-07 08:21:00+00:00</td>\n",
+       "      <td>user</td>\n",
+       "      <td>C</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>11</th>\n",
+       "      <td>2023-06-07 08:21:30+00:00</td>\n",
+       "      <td>assistant</td>\n",
+       "      <td>Correct! Option C is the correct answer...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>12</th>\n",
+       "      <td>2023-06-07 08:22:00+00:00</td>\n",
+       "      <td>assistant</td>\n",
+       "      <td>Question 4:\\nWhich financial statement provide...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13</th>\n",
+       "      <td>2023-06-07 08:22:30+00:00</td>\n",
+       "      <td>user</td>\n",
+       "      <td>C</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>14</th>\n",
+       "      <td>2023-06-07 08:23:00+00:00</td>\n",
+       "      <td>assistant</td>\n",
+       "      <td>Correct! Option C is the correct answer...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>15</th>\n",
+       "      <td>2023-06-07 08:23:30+00:00</td>\n",
+       "      <td>assistant</td>\n",
+       "      <td>Question 5:\\nWhat is the purpose of the matchi...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>16</th>\n",
+       "      <td>2023-06-07 08:24:00+00:00</td>\n",
+       "      <td>user</td>\n",
+       "      <td>B</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>17</th>\n",
+       "      <td>2023-06-07 08:24:30+00:00</td>\n",
+       "      <td>assistant</td>\n",
+       "      <td>Correct! Option B is the correct answer...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>\n",
+       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e24b7014-4d98-4fc5-9ff1-07fa5c26ba5e')\"\n",
+       "              title=\"Convert this dataframe to an interactive table.\"\n",
+       "              style=\"display:none;\">\n",
+       "        \n",
+       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
+       "       width=\"24px\">\n",
+       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
+       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
+       "  </svg>\n",
+       "      </button>\n",
+       "      \n",
+       "  <style>\n",
+       "    .colab-df-container {\n",
+       "      display:flex;\n",
+       "      flex-wrap:wrap;\n",
+       "      gap: 12px;\n",
+       "    }\n",
+       "\n",
+       "    .colab-df-convert {\n",
+       "      background-color: #E8F0FE;\n",
+       "      border: none;\n",
+       "      border-radius: 50%;\n",
+       "      cursor: pointer;\n",
+       "      display: none;\n",
+       "      fill: #1967D2;\n",
+       "      height: 32px;\n",
+       "      padding: 0 0 0 0;\n",
+       "      width: 32px;\n",
+       "    }\n",
+       "\n",
+       "    .colab-df-convert:hover {\n",
+       "      background-color: #E2EBFA;\n",
+       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
+       "      fill: #174EA6;\n",
+       "    }\n",
+       "\n",
+       "    [theme=dark] .colab-df-convert {\n",
+       "      background-color: #3B4455;\n",
+       "      fill: #D2E3FC;\n",
+       "    }\n",
+       "\n",
+       "    [theme=dark] .colab-df-convert:hover {\n",
+       "      background-color: #434B5C;\n",
+       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
+       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
+       "      fill: #FFFFFF;\n",
+       "    }\n",
+       "  </style>\n",
+       "\n",
+       "      <script>\n",
+       "        const buttonEl =\n",
+       "          document.querySelector('#df-e24b7014-4d98-4fc5-9ff1-07fa5c26ba5e button.colab-df-convert');\n",
+       "        buttonEl.style.display =\n",
+       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
+       "\n",
+       "        async function convertToInteractive(key) {\n",
+       "          const element = document.querySelector('#df-e24b7014-4d98-4fc5-9ff1-07fa5c26ba5e');\n",
+       "          const dataTable =\n",
+       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
+       "                                                     [key], {});\n",
+       "          if (!dataTable) return;\n",
+       "\n",
+       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
+       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
+       "            + ' to learn more about interactive tables.';\n",
+       "          element.innerHTML = '';\n",
+       "          dataTable['output_type'] = 'display_data';\n",
+       "          await google.colab.output.renderOutput(dataTable, element);\n",
+       "          const docLink = document.createElement('div');\n",
+       "          docLink.innerHTML = docLinkHtml;\n",
+       "          element.appendChild(docLink);\n",
+       "        }\n",
+       "      </script>\n",
+       "    </div>\n",
+       "  </div>\n",
+       "  "
+      ],
+      "text/plain": [
+       "                   timestamp     author  \\\n",
+       "0  2023-06-07 08:16:00+00:00  assistant   \n",
+       "1  2023-06-07 08:16:30+00:00       user   \n",
+       "2  2023-06-07 08:17:00+00:00  assistant   \n",
+       "3  2023-06-07 08:17:30+00:00  assistant   \n",
+       "4  2023-06-07 08:18:00+00:00       user   \n",
+       "5  2023-06-07 08:18:30+00:00  assistant   \n",
+       "6  2023-06-07 08:19:00+00:00  assistant   \n",
+       "7  2023-06-07 08:19:30+00:00       user   \n",
+       "8  2023-06-07 08:20:00+00:00  assistant   \n",
+       "9  2023-06-07 08:20:30+00:00  assistant   \n",
+       "10 2023-06-07 08:21:00+00:00       user   \n",
+       "11 2023-06-07 08:21:30+00:00  assistant   \n",
+       "12 2023-06-07 08:22:00+00:00  assistant   \n",
+       "13 2023-06-07 08:22:30+00:00       user   \n",
+       "14 2023-06-07 08:23:00+00:00  assistant   \n",
+       "15 2023-06-07 08:23:30+00:00  assistant   \n",
+       "16 2023-06-07 08:24:00+00:00       user   \n",
+       "17 2023-06-07 08:24:30+00:00  assistant   \n",
+       "\n",
+       "                                              message  \n",
+       "0   Question 1:\\nWhich of the following statements...  \n",
+       "1                                                   C  \n",
+       "2          Correct! Option C is the correct answer...  \n",
+       "3   Question 2:\\nWhich of the following expenses a...  \n",
+       "4                                                   A  \n",
+       "5          I'm sorry, but your answer is incorrect...  \n",
+       "6   Question 2 (Revised):\\nWhich of the following ...  \n",
+       "7                                                   D  \n",
+       "8          Correct! Option D is the correct answer...  \n",
+       "9   Question 3:\\nWhat is the purpose of capitalizi...  \n",
+       "10                                                  C  \n",
+       "11         Correct! Option C is the correct answer...  \n",
+       "12  Question 4:\\nWhich financial statement provide...  \n",
+       "13                                                  C  \n",
+       "14         Correct! Option C is the correct answer...  \n",
+       "15  Question 5:\\nWhat is the purpose of the matchi...  \n",
+       "16                                                  B  \n",
+       "17         Correct! Option B is the correct answer...  "
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df = pd.read_json('demo_json.json')\n",
+    "pd.read_json('demo_json.json')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {
+    "id": "anSNlvqlXh6i"
+   },
+   "outputs": [],
+   "source": [
+    "openai.api_key = \"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\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 627
+    },
+    "id": "udujJrX6SryU",
+    "outputId": "9b182162-7c1c-4d5a-be56-16947ddcda33"
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <div id=\"df-5123f950-1dca-46a6-be4d-dab5de1f8899\">\n",
+       "    <div class=\"colab-df-container\">\n",
+       "      <div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Question</th>\n",
+       "      <th>Correct Answer</th>\n",
+       "      <th>User Answer</th>\n",
+       "      <th>Evaluation</th>\n",
+       "      <th>Score</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Question 1:\\nWhich of the following statements...</td>\n",
+       "      <td>C</td>\n",
+       "      <td>C</td>\n",
+       "      <td>correct.</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>Question 2 (Revised):\\nWhich of the following ...</td>\n",
+       "      <td>D</td>\n",
+       "      <td>D</td>\n",
+       "      <td>incorrect. the correct answer is d, software d...</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>Question 3:\\nWhat is the purpose of capitalizi...</td>\n",
+       "      <td>C</td>\n",
+       "      <td>C</td>\n",
+       "      <td>incorrect. the correct answer is b.</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>Question 4:\\nWhich financial statement provide...</td>\n",
+       "      <td>C</td>\n",
+       "      <td>C</td>\n",
+       "      <td>correct</td>\n",
+       "      <td>2</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>Question 5:\\nWhat is the purpose of the matchi...</td>\n",
+       "      <td>B</td>\n",
+       "      <td>B</td>\n",
+       "      <td>correct</td>\n",
+       "      <td>3</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>\n",
+       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-5123f950-1dca-46a6-be4d-dab5de1f8899')\"\n",
+       "              title=\"Convert this dataframe to an interactive table.\"\n",
+       "              style=\"display:none;\">\n",
+       "        \n",
+       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
+       "       width=\"24px\">\n",
+       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
+       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
+       "  </svg>\n",
+       "      </button>\n",
+       "      \n",
+       "  <style>\n",
+       "    .colab-df-container {\n",
+       "      display:flex;\n",
+       "      flex-wrap:wrap;\n",
+       "      gap: 12px;\n",
+       "    }\n",
+       "\n",
+       "    .colab-df-convert {\n",
+       "      background-color: #E8F0FE;\n",
+       "      border: none;\n",
+       "      border-radius: 50%;\n",
+       "      cursor: pointer;\n",
+       "      display: none;\n",
+       "      fill: #1967D2;\n",
+       "      height: 32px;\n",
+       "      padding: 0 0 0 0;\n",
+       "      width: 32px;\n",
+       "    }\n",
+       "\n",
+       "    .colab-df-convert:hover {\n",
+       "      background-color: #E2EBFA;\n",
+       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
+       "      fill: #174EA6;\n",
+       "    }\n",
+       "\n",
+       "    [theme=dark] .colab-df-convert {\n",
+       "      background-color: #3B4455;\n",
+       "      fill: #D2E3FC;\n",
+       "    }\n",
+       "\n",
+       "    [theme=dark] .colab-df-convert:hover {\n",
+       "      background-color: #434B5C;\n",
+       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
+       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
+       "      fill: #FFFFFF;\n",
+       "    }\n",
+       "  </style>\n",
+       "\n",
+       "      <script>\n",
+       "        const buttonEl =\n",
+       "          document.querySelector('#df-5123f950-1dca-46a6-be4d-dab5de1f8899 button.colab-df-convert');\n",
+       "        buttonEl.style.display =\n",
+       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
+       "\n",
+       "        async function convertToInteractive(key) {\n",
+       "          const element = document.querySelector('#df-5123f950-1dca-46a6-be4d-dab5de1f8899');\n",
+       "          const dataTable =\n",
+       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
+       "                                                     [key], {});\n",
+       "          if (!dataTable) return;\n",
+       "\n",
+       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
+       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
+       "            + ' to learn more about interactive tables.';\n",
+       "          element.innerHTML = '';\n",
+       "          dataTable['output_type'] = 'display_data';\n",
+       "          await google.colab.output.renderOutput(dataTable, element);\n",
+       "          const docLink = document.createElement('div');\n",
+       "          docLink.innerHTML = docLinkHtml;\n",
+       "          element.appendChild(docLink);\n",
+       "        }\n",
+       "      </script>\n",
+       "    </div>\n",
+       "  </div>\n",
+       "  "
+      ],
+      "text/plain": [
+       "                                            Question Correct Answer  \\\n",
+       "0  Question 1:\\nWhich of the following statements...              C   \n",
+       "1  Question 2 (Revised):\\nWhich of the following ...              D   \n",
+       "2  Question 3:\\nWhat is the purpose of capitalizi...              C   \n",
+       "3  Question 4:\\nWhich financial statement provide...              C   \n",
+       "4  Question 5:\\nWhat is the purpose of the matchi...              B   \n",
+       "\n",
+       "  User Answer                                         Evaluation  Score  \n",
+       "0           C                                           correct.      1  \n",
+       "1           D  incorrect. the correct answer is d, software d...      1  \n",
+       "2           C                incorrect. the correct answer is b.      1  \n",
+       "3           C                                            correct      2  \n",
+       "4           B                                            correct      3  "
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Initialize necessary variables\n",
+    "prompt = \"\"\n",
+    "question = \"\"\n",
+    "correct_answer = \"\"\n",
+    "user_answer = \"\"\n",
+    "\n",
+    "# Initialize score\n",
+    "score = 0\n",
+    "\n",
+    "# Initialize an empty list to hold row data\n",
+    "row_data = []\n",
+    "\n",
+    "for index, row in df.iterrows():\n",
+    "    author = row['author']\n",
+    "    message = row['message']\n",
+    "\n",
+    "    # Choose the appropriate prompt based on the author\n",
+    "    if author == 'assistant':\n",
+    "        if 'Question' in message:\n",
+    "            question = message\n",
+    "            user_answer = ''  # Reset user_answer after a new question\n",
+    "        elif 'Correct! Option' in message:\n",
+    "            correct_answer = message.split('Option ')[1][0]\n",
+    "            if user_answer:  # If user_answer exists, make the API call\n",
+    "                prompt = f\"Given the following question:\\n{question}\\nThe student responded with: {user_answer}\\nIs the student's response correct or incorrect?\"\n",
+    "\n",
+    "                # Make an API call to OpenAI\n",
+    "                api_response = openai.Completion.create(\n",
+    "                    engine='text-davinci-003',\n",
+    "                    prompt=prompt,\n",
+    "                    max_tokens=100,\n",
+    "                    temperature=0.7,\n",
+    "                    n=1,\n",
+    "                    stop=None\n",
+    "                )\n",
+    "\n",
+    "                # Extract and evaluate the generated response\n",
+    "                generated_response = api_response.choices[0].text.strip().lower()\n",
+    "\n",
+    "                # Update score based on generated_response\n",
+    "                if 'correct' in generated_response and 'incorrect' not in generated_response:\n",
+    "                    score += 1\n",
+    "\n",
+    "                # Create a dictionary for the current row\n",
+    "                row_dict = {\n",
+    "                    'Question': question,\n",
+    "                    'Correct Answer': correct_answer,\n",
+    "                    'User Answer': user_answer,\n",
+    "                    'Evaluation': generated_response,\n",
+    "                    'Score': score\n",
+    "                }\n",
+    "                # Append the row dictionary to row_data\n",
+    "                row_data.append(row_dict)\n",
+    "\n",
+    "    elif author == 'user':\n",
+    "        user_answer = message\n",
+    "\n",
+    "# Create a DataFrame from row_data\n",
+    "output_df = pd.DataFrame(row_data)\n",
+    "output_df\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "colab": {
+   "authorship_tag": "ABX9TyOn+FniXzrkHNKH5uAKgyUD",
+   "include_colab_link": true,
+   "provenance": []
+  },
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.16"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

lo-achievement/instructor_intr_notebook.ipynb

@@ -0,0 +1,3153 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "collapsed": true,
+    "id": "brzvVeAsYiG2"
+   },
+   "source": [
+    "<a href=\"https://colab.research.google.com/github/vanderbilt-data-science/lo-achievement/blob/main/instructor_intr_notebook.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "id": "WMKrKfx8_3fc"
+   },
+   "source": [
+    "# Instructor Grading and Assessment\n",
+    "This notebook executes grading of student submissions of chats with ChatGPT, exported in JSON. Run each cell should be run in order, and follow the prompts displayed when appropriate."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 35,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 16
+    },
+    "id": "696FqPrTYiG3",
+    "outputId": "9679a415-8ab7-4c5f-e715-954d6801b6ec"
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "import ipywidgets as widgets\n",
+    "from IPython.display import display, HTML, clear_output\n",
+    "import io\n",
+    "import zipfile\n",
+    "import os\n",
+    "import json\n",
+    "import pandas as pd\n",
+    "import glob\n",
+    "from getpass import getpass"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 36,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 16
+    },
+    "id": "fTlnrMwmYiG4",
+    "outputId": "e811e000-e9ec-43b6-d136-59d5134adeaf"
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "# \"global\" variables modified by mutability\n",
+    "grade_settings = {'learning_objectives':None,\n",
+    "                  'json_file_path':None,\n",
+    "                  'json_files':None }"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "id": "jb0jnIE14Vuh"
+   },
+   "source": [
+    "The `InstructorGradingConfig` holds the contents of the instantiated object including making graindg settings, extracting files from a zip archive, loading JSON files into DataFrames, and displaying relevant information in the output widget."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 37,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 16
+    },
+    "id": "mPLdaWiuYiG4",
+    "outputId": "7a698bc1-7954-44ac-83c8-71d1dc410749"
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "class InstructorGradingConfig:\n",
+    "    def __init__(self):\n",
+    "        # layouts to help with styling\n",
+    "        self.items_layout = widgets.Layout(width='auto')\n",
+    "\n",
+    "        self.box_layout = widgets.Layout(display='flex',\n",
+    "                                          flex_flow='column',\n",
+    "                                          align_items='stretch',\n",
+    "                                          width='50%',\n",
+    "                                          border='solid 1px gray',\n",
+    "                                          padding='0px 30px 20px 30px')\n",
+    "\n",
+    "        # Create all components\n",
+    "        self.ui_title = widgets.HTML(value=\"<h2>Instructor Grading Configuration</h2>\")\n",
+    "\n",
+    "        self.run_button = widgets.Button(description='Submit', button_style='success', icon='check')\n",
+    "        self.status_output = widgets.Output()\n",
+    "        self.status_output.append_stdout('Waiting...')\n",
+    "\n",
+    "        # Setup click behavior\n",
+    "        self.run_button.on_click(self._setup_environment)\n",
+    "\n",
+    "        # Reset rest of state\n",
+    "        self.reset_state()\n",
+    "\n",
+    "    def reset_state(self, close_all=False):\n",
+    "\n",
+    "        if close_all:\n",
+    "            self.learning_objectives_text.close()\n",
+    "            self.file_upload.close()\n",
+    "            self.file_upload_box.close()\n",
+    "            #self.ui_container.close()\n",
+    "\n",
+    "        self.learning_objectives_text = widgets.Textarea(value='', description='Learning Objectives',\n",
+    "                                                         placeholder='Learning objectives: 1. Understand and implement classes in object-oriented programming',\n",
+    "                                                         layout=self.items_layout,\n",
+    "                                                         style={'description_width': 'initial'})\n",
+    "        self.file_upload = widgets.FileUpload(\n",
+    "            accept='.zip',  # Accepted file extension e.g. '.txt', '.pdf', 'image/*', 'image/*,.pdf'\n",
+    "            multiple=False  # True to accept multiple files upload else False\n",
+    "        )\n",
+    "        self.file_upload_box = widgets.HBox([widgets.Label('Upload User Files:\\t'), self.file_upload])\n",
+    "\n",
+    "\n",
+    "        # Create a VBox container to arrange the widgets vertically\n",
+    "        self.ui_container = widgets.VBox([self.ui_title, self.learning_objectives_text,\n",
+    "                                           self.file_upload_box, self.run_button, self.status_output],\n",
+    "                                          layout=self.box_layout)\n",
+    "\n",
+    "\n",
+    "    def _setup_environment(self, btn):\n",
+    "        grade_settings['learning_objectives'] = self.learning_objectives_text.value\n",
+    "        grade_settings['json_file_path'] = self.file_upload.value\n",
+    "\n",
+    "        if self.file_upload.value:\n",
+    "            try:\n",
+    "                input_file = list(self.file_upload.value.values())[0]\n",
+    "                extracted_zip_dir = list(grade_settings['json_file_path'].keys())[0][:-4]\n",
+    "            except:\n",
+    "                input_file = self.file_upload.value[0]\n",
+    "                extracted_zip_dir = self.file_upload.value[0]['name'][:-4]\n",
+    "\n",
+    "            self.status_output.clear_output()\n",
+    "            self.status_output.append_stdout('Loading zip file...\\n')\n",
+    "\n",
+    "            with zipfile.ZipFile(io.BytesIO(input_file['content']), \"r\") as z:\n",
+    "                z.extractall()\n",
+    "                extracted_files = z.namelist()\n",
+    "\n",
+    "            self.status_output.append_stdout('Extracted files and directories: {0}\\n'.format(', '.join(extracted_files)))\n",
+    "\n",
+    "            # load all json files\n",
+    "            grade_settings['json_files'] = glob.glob(''.join([extracted_zip_dir, '/**/*.json']), recursive=True)\n",
+    "\n",
+    "            #status_output.clear_output()\n",
+    "            self.status_output.append_stdout('Loading successful!\\nLearning Objectives: {0}\\nExtracted JSON files: {1}'.format(grade_settings['learning_objectives'],\n",
+    "                                                                                                        ', '.join(grade_settings['json_files'])))\n",
+    "\n",
+    "        else:\n",
+    "            self.status_output.clear_output()\n",
+    "            self.status_output.append_stdout('Please upload a zip file.')\n",
+    "\n",
+    "        # Clear values so they're not saved\n",
+    "        self.learning_objectives_text.value = ''\n",
+    "        self.reset_state(close_all=True)\n",
+    "        self.run_ui_container()\n",
+    "\n",
+    "        with self.status_output:\n",
+    "            print('Extracted files and directories: {0}\\n'.format(', '.join(extracted_files)))\n",
+    "            print('Loading successful!\\nLearning Objectives: {0}\\nExtracted JSON files: {1}'.format(grade_settings['learning_objectives'],\n",
+    "                                                                                                        ', '.join(grade_settings['json_files'])))\n",
+    "            print('Submitted and Reset all values.')\n",
+    "\n",
+    "\n",
+    "    def run_ui_container(self):\n",
+    "        display(self.ui_container, clear=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 16
+    },
+    "id": "4wCQ4Wk8YiG4",
+    "outputId": "5c602e80-a210-4449-fd6c-eb8bf3213407"
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "#This code helps in the case that we have problems with metadata being retained.\n",
+    "#!jupyter nbconvert --ClearOutputPreprocessor.enabled=True --ClearMetadataPreprocessor.enabled=True --ClearMetadataPreprocessor.preserve_cell_metadata_mask \"colab\" --ClearMetadataPreprocessor.preserve_cell_metadata_mask \"kernelspec\" --ClearMetadataPreprocessor.preserve_cell_metadata_mask \"language_info\" --to=notebook --output=instructor_inst_notebook.ipynb instructor_intr_notebook.ipynb"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "id": "gj1K3MjHDlqb"
+   },
+   "source": [
+    "# User Settings and Submission Upload\n",
+    "The following two cells will ask you for your OpenAI API credentials and to upload the json file of the student submission."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 519,
+     "referenced_widgets": [
+      "a84d31fb8f4e4bafb74035158834b404",
+      "b051a90758434644955747bc02d00bab",
+      "252b8009f3734ed2908049ebb40c0247",
+      "6622f76f91f44527a87a7575bbd388d2",
+      "654ab6d155eb457ea5c719a9ac27ad5b",
+      "86cb4f568f454ff8832face502fb0745",
+      "e30fe87f01bc4580a61713b5b72439a2",
+      "d16b25c7e9e948938c9303fbe8ae3dcc",
+      "453b12da4b6540cd9e4e57f73a4d670c",
+      "b74cf92175374028948d4cf529d4d1e6",
+      "f7d75b0a32554a9589c513336fc30095",
+      "7f7164e80a464ba9b99f96c10132db25",
+      "49f80567705147f0b82d45b7f06dd1ba",
+      "5a17f4509d194105b23dd616e45183d5",
+      "81c4dda35a7d4e15821bb4bc0973354e",
+      "df1c46361f714aceb9c046f98fede40c",
+      "60b80d550efa403a825a3cb913c26f53",
+      "d0bd0e3f12594ff1a51365b65a3fcc43",
+      "dfa8d6c7d70b42468cbda035de89404c",
+      "26d13984d45745858d3b890bc7f18a90",
+      "53722998fbe64a7c94829b79e8cd69d6",
+      "1b7ee0de15484cd5aecd6d8ca3b6ee9d",
+      "dde20647d3594d31b66b19659f53a95e",
+      "8610fffd2d2a4ec28f8c874c06073ce7",
+      "54e3918921f44fb4a9020beab951fcdf",
+      "1072a8a142f64dfd96ee528a2e9d1595",
+      "67b4083cd4234f52bb7cca27ab9cddb3",
+      "d0a1ebdf7fc0473f91c39b29ca580934",
+      "abbecdc637694e7cb026e003244e7037",
+      "7f814595d31e4b86992b5bd6bc85ced4",
+      "76548751bb9c4bcb9d4f39788ea7d4af",
+      "dbb88901f5084d49af208b91b52b6073"
+     ]
+    },
+    "id": "oQOeYl9OYiG5",
+    "outputId": "bb5b7dc4-ea7b-41ea-e741-2fb2bf66cccc"
+   },
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "1b7ee0de15484cd5aecd6d8ca3b6ee9d",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "VBox(children=(HTML(value='<h2>Instructor Grading Configuration</h2>'), Textarea(value='', description='Learni…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "InstructorGradingConfig().run_ui_container()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "id": "W9SqmkpeIgpk"
+   },
+   "source": [
+    "You will need an OpenAI API key in order to access the chat functionality. In the following cell, you'll see a blank box pop up - copy your API key there and press enter."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 32
+    },
+    "id": "MK8R5DmEYiG5",
+    "outputId": "09e11ee6-5a9f-4b61-ff61-ddf82a68c498"
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "··········\n"
+     ]
+    }
+   ],
+   "source": [
+    "# setup open AI api key\n",
+    "openai_api_key = getpass()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "collapsed": true,
+    "id": "0bp158bj_0s6"
+   },
+   "source": [
+    "# Execute Grading\n",
+    "Run this cell set to have the generative AI assist you in grading."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "id": "vyJuQ7RUR8tB"
+   },
+   "source": [
+    "## Installation and Loading"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 16
+    },
+    "id": "tjKxWLA3YiG5",
+    "outputId": "6dc85dff-4baa-44f0-edef-42925e6c271a"
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "%%capture\n",
+    "# install additional packages if needed\n",
+    "! pip install -q langchain openai"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 16
+    },
+    "id": "S3oQiNm_YiG5",
+    "outputId": "95e43744-fadc-45cc-bb02-05737d12fcb2"
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "# import necessary libraries here\n",
+    "from langchain.llms import OpenAI\n",
+    "from langchain.chat_models import ChatOpenAI\n",
+    "from langchain.prompts import PromptTemplate\n",
+    "from langchain.document_loaders import TextLoader\n",
+    "from langchain.indexes import VectorstoreIndexCreator\n",
+    "from langchain.text_splitter import CharacterTextSplitter\n",
+    "from langchain.embeddings import OpenAIEmbeddings\n",
+    "from langchain.schema import SystemMessage, HumanMessage, AIMessage\n",
+    "import openai"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 16
+    },
+    "id": "uXfSTQPrYiG5",
+    "outputId": "f85a6c4a-009f-4b30-f74a-04b6bfd85af6"
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "# Helper because lines are printed too long; helps with wrapping visualization\n",
+    "from IPython.display import HTML, display\n",
+    "\n",
+    "def set_css():\n",
+    "  display(HTML('''\n",
+    "  <style>\n",
+    "    pre {\n",
+    "        white-space: pre-wrap;\n",
+    "    }\n",
+    "  </style>\n",
+    "  '''))\n",
+    "get_ipython().events.register('pre_run_cell', set_css)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 16
+    },
+    "id": "sTQFW9TxYiG5",
+    "outputId": "e291e167-e635-4006-965d-29fe1a0db10f"
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "# Set pandas display options\n",
+    "pd.set_option('display.max_columns', None)\n",
+    "pd.set_option('display.max_colwidth', 0)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "id": "DOACT_LSSM58"
+   },
+   "source": [
+    "Setting of API key in environment and other settings"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 16
+    },
+    "id": "OV05xRtDYiG5",
+    "outputId": "0d6339d9-bc32-49e9-955f-99947b510456"
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "#extract info from dictionary\n",
+    "json_file_path = grade_settings['json_file_path']\n",
+    "learning_objectives = grade_settings['learning_objectives']\n",
+    "\n",
+    "#set API key\n",
+    "os.environ[\"OPENAI_API_KEY\"] = openai_api_key\n",
+    "openai.api_key = openai_api_key"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "id": "YreIs-I-tuxx"
+   },
+   "source": [
+    "Initiate the OpenAI model using Langchain."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 16
+    },
+    "id": "ZRn9wbJBYiG5",
+    "outputId": "c09c7a7c-1ca0-4860-b39d-91778f183307"
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "llm = ChatOpenAI(model='gpt-3.5-turbo-16k')\n",
+    "messages = [\n",
+    "    SystemMessage(content=\"You are a helpful assistant.\"),\n",
+    "    HumanMessage(content=\"\")\n",
+    "]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "id": "pIKYtr0UTJNc"
+   },
+   "source": [
+    "## Functions to help with loading json"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "id": "t7O3XPC29Osw"
+   },
+   "source": [
+    "`file_upload_json_to_df` helps when you use the file uploader as the json is directly read in this case. `clean_keys` helps when there are errors on the keys when reading."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 16
+    },
+    "id": "qGxPHexrYiG5",
+    "outputId": "80657bb1-97e8-423a-a2ff-99afe8d22718"
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "# Strip beginning and ending newlines\n",
+    "def clean_keys(loaded_json):\n",
+    "  out_json = [{key.strip():value for key, value in json_dict.items()} for json_dict in loaded_json ]\n",
+    "  return out_json\n",
+    "\n",
+    "# Convert difficult datatypes to newlines\n",
+    "def file_upload_json_to_df(upload_json):\n",
+    "\n",
+    "  #get middle key of json to extract content\n",
+    "  fname = list(upload_json.keys())[0]\n",
+    "\n",
+    "  #load the json; strict allows us to get around encoding issues\n",
+    "  loaded_json = json.loads(upload_json[fname]['content'], strict=False)\n",
+    "\n",
+    "  #clean the keys if needed\n",
+    "  loaded_json = clean_keys(loaded_json)\n",
+    "\n",
+    "  return pd.DataFrame(loaded_json)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "id": "N2yuYFQJYiG6"
+   },
+   "source": [
+    "`create_user_dataframe` filters based on role to create a dataframe for only user responses"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 17
+    },
+    "id": "58hygjTXYiG6",
+    "outputId": "8f3683fb-f3da-45f3-8338-772e7583d4cc"
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "def create_user_dataframe(df):\n",
+    "  df_user = df.query(\"`author` == 'user'\")\n",
+    "\n",
+    "  return df_user"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "id": "MOwaLI97Igpm"
+   },
+   "source": [
+    "`load_json_as_df` helps when you use the file uploader as the json is directly read in this case. It accepts the path to the JSON to load the dataframe based on the json."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 131,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 16
+    },
+    "id": "w0xN9CJeYiG6",
+    "outputId": "9422452f-9a97-4f22-9e49-ea0812c298fd"
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "def load_json_as_df(fpath):\n",
+    "    # check if file is .json\n",
+    "    if not fpath.endswith('.json'):\n",
+    "        return None\n",
+    "\n",
+    "    keys = [\"timestamp\", \"author\", \"message\"]\n",
+    "\n",
+    "    df_out = None\n",
+    "    out_error = None\n",
+    "\n",
+    "    try:\n",
+    "        # Read JSON file\n",
+    "        with open(fpath, \"r\") as f:\n",
+    "            json_data = f.read()\n",
+    "\n",
+    "        # Load JSON data\n",
+    "        data = json.loads(json_data, strict=False)\n",
+    "\n",
+    "        # Quick check to see if we can fix common errors in json\n",
+    "        # 1. JSON responses wrapped in enclosing dictionary\n",
+    "        if isinstance(data, dict):\n",
+    "            if len(data.keys()) == 1:\n",
+    "                data = data[list(data.keys())[0]]\n",
+    "            else:\n",
+    "                data = [data]  # convert to list otherwise\n",
+    "\n",
+    "        # We only operate on lists of dictionaries\n",
+    "        if isinstance(data, list):\n",
+    "            data = clean_keys(data)  # clean keys to make sure there are no unnecessary newlines\n",
+    "\n",
+    "            if all(all(k in d for k in keys) for d in data):\n",
+    "                # Filter only the student messages based on the \"author\" key\n",
+    "                data = [d for d in data if d[\"author\"].lower() == \"user\"]\n",
+    "\n",
+    "                df_out = pd.json_normalize(data)\n",
+    "                if len(df_out) <= 1:\n",
+    "                    out_error = [fpath, \"Warning: JSON keys correct, but something wrong with the overall structure of the JSON when converting to the dataframe. The dataframe only has one row. Skipping.\"]\n",
+    "                    df_out = None\n",
+    "            else:\n",
+    "                out_error = [fpath, \"Error: JSON Keys are incorrect. Found keys: \" + str(list(data[0].keys()))]\n",
+    "        else:\n",
+    "            out_error = [fpath, \"Error: Something is wrong with the structure of the JSON.\"]\n",
+    "\n",
+    "    except Exception as e:\n",
+    "        print(f\"Error processing file {fpath}: {str(e)}\")\n",
+    "        out_error = [fpath, \"Fatal System Error: \" + str(e)]\n",
+    "\n",
+    "    if df_out is not None:\n",
+    "        df_out['filename'] = fpath\n",
+    "\n",
+    "    return df_out, out_error"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "id": "N2yuYFQJYiG6"
+   },
+   "source": [
+    "`create_user_dataframe` filters based on role to create a dataframe for only user responses"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 132,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 16
+    },
+    "id": "58hygjTXYiG6",
+    "outputId": "44b588d6-6b6c-4c9e-b944-62ac29117344"
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "def create_user_dataframe(df):\n",
+    "  df_user = df.query(\"`author` == 'user'\")\n",
+    "\n",
+    "  return df_user"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "id": "KA5moX-1Igpn"
+   },
+   "source": [
+    "The `process_file` and `process_files` functions provide the implementation of prompt templates for instructor grading. It uses the input components to assemble a prompt and then sends this prompt to the llm for evaluation alongside the read dataframes."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 16
+    },
+    "id": "nFz3UVL3YiG6",
+    "outputId": "01029e5f-b313-4941-d572-6dca5903d4ac"
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "def process_file(df, desc, instr, print_results):\n",
+    "    messages_as_string = '\\n'.join(df['message'].astype(str))\n",
+    "    context = messages_as_string\n",
+    "\n",
+    "    # Assemble prompt\n",
+    "    prompt = desc if desc is not None else \"\"\n",
+    "    prompt = (prompt + instr + \"\\n\") if instr is not None else prompt\n",
+    "    prompt = prompt + \"Here is the chat log: \\n\\n\" + context + \"\\n\"\n",
+    "\n",
+    "    # Get results and optionally print\n",
+    "    messages[1] = HumanMessage(content=prompt)\n",
+    "    result = llm(messages)\n",
+    "\n",
+    "    # Check if 'filename' exists in df\n",
+    "    if 'filename' in df:\n",
+    "        if print_results:\n",
+    "            print(f\"\\n\\nResult for file {df['filename'][0]}: \\n{result.content}\")\n",
+    "    else:\n",
+    "        if print_results:\n",
+    "            print(f\"\\n\\nResult for file: Unknown Filename \\n{result.content}\")\n",
+    "\n",
+    "    return result\n",
+    "\n",
+    "def process_files(json_dfs, output_desc=None, grad_instructions=None, use_defaults = False, print_results=True):\n",
+    "    if use_defaults:\n",
+    "        output_desc = (\"Given the following chat log, create a table with the question number, the question content, answer, \"\n",
+    "                       \"whether or not the student answered correctly on the first try, and the number of attempts it took to get the right answer. \")\n",
+    "        grad_instructions = (\"Then, calculate the quiz grade from the total number of assessment questions. \"\n",
+    "                             \"Importantly, a point should only be granted if an answer was correct on the very first attempt. \"\n",
+    "                             \"If an answer was not correct on the first attempt, even if it was correct in subsequent attempts, no point should be awarded for that question. \")\n",
+    "\n",
+    "    results = [process_file(df, output_desc, grad_instructions, print_results) for df in json_dfs]\n",
+    "\n",
+    "    return results"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 17
+    },
+    "id": "EhryP8utrR9D",
+    "outputId": "51f4a60d-d6b7-4885-85c4-410c741ed651"
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "def output_log_file(df_list, results_list, log_file='evaluation_log.txt'):\n",
+    "    \"\"\"\n",
+    "    Create a single log file containing evaluation results for all students.\n",
+    "\n",
+    "    Parameters:\n",
+    "        df_list (list of pandas.DataFrame): List of DataFrames.\n",
+    "        results_list (list of ai_model_response): List of evaluation results.\n",
+    "        log_file (str): File name where the evaluation log will be saved. Default is 'evaluation_log.txt'.\n",
+    "\n",
+    "    Returns:\n",
+    "        None\n",
+    "    \"\"\"\n",
+    "    with open(log_file, 'w') as log:\n",
+    "        for df, result in zip(df_list, results_list):\n",
+    "            log.write(f\"File: {df['filename'][0]}\\n\")\n",
+    "            log.write(result.content)\n",
+    "            log.write(\"\\n\\n\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "id": "lXQ45cJ1AztR"
+   },
+   "source": [
+    "`pretty_print` makes dataframes look better when printed by substituting non-HTML with HTML for rendering."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 134,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 16
+    },
+    "id": "0te_RLOOYiG6",
+    "outputId": "1dc53c98-5f68-4902-b377-7bba451395f0"
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "def pretty_print(df):\n",
+    "    return display( HTML( df.to_html().replace(\"\\\\n\",\"<br>\") ) )"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "id": "I3rKk7lJYiG6"
+   },
+   "source": [
+    "`save_as_csv` saves the dataframe as a CSV"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 135,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 16
+    },
+    "id": "DnrH2ldeYiG6",
+    "outputId": "f1f6153b-49db-4145-c188-373685ffdcf4"
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "def save_as_csv(df, file_name):\n",
+    "  df.to_csv(file_name, index=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 136,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 16
+    },
+    "id": "Vgo_y8R8bzTE",
+    "outputId": "f5effec2-8620-4c1d-be15-672b8cb3de21"
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "def show_json_loading_errors(err_list):\n",
+    "  if err_list:\n",
+    "    print(\"The following files have the following errors upon loading and will NOT be processed:\", '\\n'.join(err_list))\n",
+    "  else:\n",
+    "    print(\"No errors found in uploaded zip JSON files.\")\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "id": "85h5oTysJkHs"
+   },
+   "source": [
+    "## Final data preparation steps"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 137,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 16
+    },
+    "id": "Upah5_ygZRZx",
+    "outputId": "639a0d70-6a93-4462-f65a-2e24f30643c0"
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "#additional processing setup\n",
+    "json_files = grade_settings['json_files']\n",
+    "load_responses = [load_json_as_df(jf) for jf in json_files]\n",
+    "\n",
+    "#unzip to two separate lists\n",
+    "all_json_dfs, errors_list = zip(*load_responses)\n",
+    "\n",
+    "# Remove failed JSONs\n",
+    "all_json_dfs = [df for df in all_json_dfs if df is not None]\n",
+    "\n",
+    "# Update errors list to be individual strings\n",
+    "errors_list = [' '.join(err) for err in errors_list if err is not None]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "id": "P_H4uIfmAsr0"
+   },
+   "source": [
+    "# AI-Assisted Evaluation\n",
+    "Introduction and Instructions\n",
+    "--------------------------------------------------\n",
+    "The following example illustrates how you can specify important components of the prompts for sending to the llm. The `process_files` function will iterate over all of the submissions in your zip file, create dataframes of results (via instruction by setting `output_setup`), and also perform evaluation based on your instructions (via instruction by setting `grading_instructions`).\n",
+    "\n",
+    "Example functionality is demonstrated below."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 138,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 35
+    },
+    "id": "9zIPjG5lco3Z",
+    "outputId": "cc9531c0-3939-4c9f-dc6c-9d2cfa0ea7d2"
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "No errors found in uploaded zip JSON files.\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Print list of files with the incorrect format\n",
+    "show_json_loading_errors(errors_list)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 139,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 1000
+    },
+    "id": "utPzYUoKYiG9",
+    "outputId": "eb3e1769-eb7a-4c93-96bc-6c998df55ef1"
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "Result for file instructorTest/spencer-smith_jesse.json: \n",
+      "Summary and feedback for student responses:\n",
+      "\n",
+      "Student 1:\n",
+      "The student provided an excellent response to Question 1. They accurately explained the purpose of capitalizing expenses when incorporating them into the estimate of corporate earnings. They highlighted the importance of accurately reflecting the timing of costs and their related benefits, and how capitalizing expenses can impact a company's financial statements. The student also mentioned the matching principle of accounting and its role in ensuring the comparability and fairness of financial statements. Overall, the response is comprehensive and well-written. Well done!\n",
+      "\n",
+      "Student 2:\n",
+      "The student gave a great response to Question 2. They correctly stated that expenses should be capitalized when they provide value beyond the current accounting period. The student also provided examples of capital expenses, such as the purchase price of a delivery truck or the cost of a building renovation. These examples demonstrate a clear understanding of the topic. The response is well-explained and shows a good grasp of the concept. Great job!\n",
+      "\n",
+      "Numeric summary:\n",
+      "Both students provided correct answers to their respective questions, earning them a point each. Therefore, the numeric summary is as follows:\n",
+      "Student 1: 1 point\n",
+      "Student 2: 1 point\n",
+      "\n",
+      "\n",
+      "Result for file instructorTest/bell_charreau.json: \n",
+      "Summary:\n",
+      "- The first student's response inaccurately states that capitalizing expenses is done to make the money 'look good' on the earnings report. The assistant provides a detailed explanation of the correct purpose of capitalizing expenses.\n",
+      "- The second student's response partially identifies that capitalized expenses provide benefits for a longer period, but the assistant provides a more comprehensive explanation of what types of expenses should be capitalized and why they are treated differently from regular expenses.\n",
+      "\n",
+      "Feedback for the first student:\n",
+      "The student accurately identified the purpose of capitalizing expenses, but their explanation was not entirely correct. They incorrectly stated that it is done to make the money 'look good' on the earnings report. The assistant provided a clear and detailed explanation of the correct purpose of capitalizing expenses and how it aligns with accounting principles.\n",
+      "\n",
+      "Feedback for the second student:\n",
+      "The student partially identified the types of expenses that should be capitalized and why they are treated differently from regular expenses. However, their explanation was not comprehensive. The assistant provided a more detailed explanation of what types of expenses should be capitalized and why, as well as the concept of depreciation or amortization for spreading out the costs over time.\n",
+      "\n",
+      "Numeric Summary:\n",
+      "The first student's response was partially correct, so they receive 0.5 points.\n",
+      "The second student's response was also partially correct, so they receive 0.5 points.\n",
+      "The total point count is 1.\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Example\n",
+    "output_setup = (\"For each student response given in the following chat log, please generate a summary and detailed feedback for each students' responses,\"\n",
+    "                  \", including what the student did well, and what was done poorly. \"\n",
+    "                  \"Additionally, please filter feedback alphabetically by the name of the student from the filename.\")\n",
+    "grading_instructions = (\"Then, calculate a numeric summary, summing up the point totals, \"\n",
+    "                  \"in which a point is awarded for answering correctly. \")\n",
+    "\n",
+    "# Assuming `file_paths` is a list of file paths.\n",
+    "processed_submissions = process_files(all_json_dfs, output_setup, grading_instructions, use_defaults=False, print_results=True)\n",
+    "\n",
+    "output_log_file(all_json_dfs, processed_submissions)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "id": "Pc1myGweIgpo"
+   },
+   "source": [
+    "## Instructor-Specified Evaluation\n",
+    "Now, you can use the following code to create your settings. Change `output_setup` and `grading_instructions` as desired, making sure to keep the syntax (beginning and ending parentheses,and quotes at the beginning and end of each line) correct. `output_setup` has been copied from the previous cell, but you should fill in `grading_instructions`.\n",
+    "\n",
+    "### File Processing Options\n",
+    "The `process_files` function has a number of settings.\n",
+    "* The first setting must always be `all_json_dfs`, which contains the tabular representation of the json output.\n",
+    "* The other settings should be set by name, and are:\n",
+    "  * **`output_desc`**: Shown as `output_setup` here, this contains the isntructions about how you want to the tabular representation to be set up. Note that you can also leave this off of the function list (just erase it and the following comma).\n",
+    "  * **`grad_instructions`**: Shown as `grading_instructions` here, use this variable to set grading instructions. Note that you can also leave this off of the function list (erase it and the following comma)\n",
+    "  * **`use_defaults`**: Some default grading and instruction prompts have already been created. If you set `use_defaults=TRUE`, both the grading instructions and the output table description will use the default prompts provided by the program, regardless of whether you have set values for `output_desc` or `grad_instructions`.\n",
+    "  * **`print_results`**: By default, the results will be printed for all students. However, if you don't want to see this output, you can set `print_results=False`.\n",
+    "\n",
+    "Again, make sure to observe the syntax. The defaults used in the program are shown in the above example."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 34,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 16
+    },
+    "id": "GiebKVlbYiG9",
+    "outputId": "d4769d23-393c-4986-9418-5cef6944e6ab"
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "output_setup = (\"For each student response given in the following chat log, please generate a summary and detailed feedback for each students' responses,\"\n",
+    "                  \", including what the student did well, and what was done poorly. \")\n",
+    "\n",
+    "# add your own grading instructions\n",
+    "grading_instructions = (\"INSERT ANY CUSTOM GRADING INSTRUCTIONS HERE\")\n",
+    "\n",
+    "# Assuming `file_paths` is a list of file paths.\n",
+    "processed_submissions = process_files(all_json_dfs, output_setup, grading_instructions, use_defaults=False, print_results=True)\n",
+    "\n",
+    "output_log_file(all_json_dfs, processed_submissions)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "id": "snLA6OZ83CrS"
+   },
+   "source": [
+    "## Grading based on Blooms Taxonomy\n",
+    "Another mechanism of evaluation is through Bloom's Taxonomy, where student responses will be evaluated based on where they fall on Bloom's Taxonomy. The higher the score with Bloom's Taxonomy, the more depth is illustrated by the question."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 140,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 625
+    },
+    "id": "HEPXCJdrYiG-",
+    "outputId": "add813e2-6b7c-4772-dac5-5f756e893b8f"
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "Result for file 0    instructorTest/bell_charreau.json      \n",
+      "0    instructorTest/spencer-smith_jesse.json\n",
+      "Name: filename, dtype: object: \n",
+      "Student 1:\n",
+      "Summary: The student incorrectly states that capitalizing expenses is done to make the money look good on the earnings report.\n",
+      "Feedback: The student's response is not accurate. They misunderstood the purpose of capitalizing expenses. The main purpose is to spread the cost of certain long-term assets over their useful life, not to make the money 'look good' on the earnings report. \n",
+      "Overall Level of Engagement and Knowledge: 1 (Remember)\n",
+      "\n",
+      "Student 2:\n",
+      "Summary: The student partially understands the purpose of capitalizing expenses, but their answer could be more comprehensive.\n",
+      "Feedback: The student correctly notes that capitalized expenses provide benefits for a longer period and are different from regular expenses. However, their answer could be more comprehensive and provide a more thorough explanation of why certain expenses are capitalized and how they are treated differently.\n",
+      "Overall Level of Engagement and Knowledge: 3 (Apply)\n",
+      "\n",
+      "Student 3:\n",
+      "Summary: The student provides a comprehensive and accurate explanation of the purpose of capitalizing expenses.\n",
+      "Feedback: The student's response is excellent. They provide a comprehensive explanation of why capitalizing expenses is crucial for incorporating costs into the estimate of corporate earnings. They correctly highlight the timing of costs and their related benefits, the impact on financial statements, and the alignment with the matching principle of accounting.\n",
+      "Overall Level of Engagement and Knowledge: 6 (Create)\n",
+      "\n",
+      "Student 4:\n",
+      "Summary: The student accurately states that expenses should be capitalized when they provide value beyond the current accounting period and provides relevant examples.\n",
+      "Feedback: The student's response is great. They correctly identify the types of expenses that should be capitalized and provide relevant examples. They demonstrate a clear understanding of the concept and provide a well-explained answer.\n",
+      "Overall Level of Engagement and Knowledge: 5 (Evaluate)\n"
+     ]
+    }
+   ],
+   "source": [
+    "output_setup = (\"For each student response given in the following chat log, please generate a summary and detailed feedback for each students' responses,\"\n",
+    "                  \", including what the student did well, and what was done poorly. \")\n",
+    "grading_instructions = \"\"\"\\nEvaluate the each student's overall level or engagement and knowledge, based on bloom's taxonomy using their responses.\n",
+    "Bloom's taxonomy is rated on a 1-6 point system, with 1 being remember (recall facts and basic concepts), 2 being understand (explain ideas or concepts),\n",
+    "3 being apply (use information in new situations), 4 being analyze (draw connections among ideas), 5 being evaluate (justify a stand or decision),\n",
+    "and 6 being create (produce new or original work). Assign the interaction a score from 1-6, where 1 = remember, 2 = understand, 3 = apply, 4 = analyze,\n",
+    "5 = evaluate, and 6 = create.\"\"\"\n",
+    "\n",
+    "# Assuming `file_paths` is a list of file paths.\n",
+    "processed_submissions = process_files(all_json_dfs, output_setup, grading_instructions, use_defaults=False, print_results=True)\n",
+    "\n",
+    "output_log_file(all_json_dfs, processed_submissions)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "id": "FI5-vnUvXM03"
+   },
+   "source": [
+    "# Returning Results\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "id": "LgoGt82CYiG-"
+   },
+   "source": [
+    "**Extract Student Responses ONLY from CHAT JSON**\n",
+    "\n",
+    "Below are relevant user components of dataframes, including the conversion from the original json, the interaction labeled dataframe, and the output dataframe. Check to make sure they make sense."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 16
+    },
+    "id": "HVq9i_mXYiG-",
+    "outputId": "fb5251e9-a327-4dfa-e294-03f5c58f6d35"
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "def write_responses_to_csv(json_dfs):\n",
+    "    # Concatenate all dataframes in json_dfs into one large dataframe\n",
+    "    df = pd.concat(json_dfs)\n",
+    "\n",
+    "    # Write the dataframe to a CSV\n",
+    "    df.to_csv('all_student_responses.csv', index=False)\n",
+    "\n",
+    "write_responses_to_csv(all_json_dfs)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "id": "1WIGxKmDYiG-"
+   },
+   "source": [
+    "**Saving/Downloading AI-Assisted Student Evaluation from Chat JSON**\n",
+    "\n",
+    "Execute the following cell to have all of your students' data returned in a single CSV file."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 16
+    },
+    "id": "QnWNEeqjYiG-",
+    "outputId": "e98c0c39-8449-45d4-f93a-d9394f6781bf"
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "  <style>\n",
+       "    pre {\n",
+       "        white-space: pre-wrap;\n",
+       "    }\n",
+       "  </style>\n",
+       "  "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "# Start with an empty dataframe\n",
+    "all_results_df = pd.DataFrame()\n",
+    "\n",
+    "for result in processed_submissions:\n",
+    "\n",
+    "    # Append the data from the current file to the master dataframe\n",
+    "    all_results_df = pd.concat([all_results_df, df])\n",
+    "\n",
+    "# Now all_results_df contains data from all the files\n",
+    "\n",
+    "# Write all results to a single CSV\n",
+    "all_results_df.to_csv('all_results.csv', index=False)"
+   ]
+  }
+ ],
+ "metadata": {
+  "colab": {
+   "include_colab_link": true,
+   "provenance": []
+  },
+  "kernelspec": {
+   "display_name": "Python 3",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python",
+   "version": "3.10.6"
+  },
+  "widgets": {
+   "application/vnd.jupyter.widget-state+json": {
+    "1072a8a142f64dfd96ee528a2e9d1595": {
+     "model_module": "@jupyter-widgets/controls",
+     "model_module_version": "1.5.0",
+     "model_name": "LabelModel",
+     "state": {
+      "_dom_classes": [],
+      "_model_module": "@jupyter-widgets/controls",
+      "_model_module_version": "1.5.0",
+      "_model_name": "LabelModel",
+      "_view_count": null,
+      "_view_module": "@jupyter-widgets/controls",
+      "_view_module_version": "1.5.0",
+      "_view_name": "LabelView",
+      "description": "",
+      "description_tooltip": null,
+      "layout": "IPY_MODEL_abbecdc637694e7cb026e003244e7037",
+      "placeholder": "​",
+      "style": "IPY_MODEL_7f814595d31e4b86992b5bd6bc85ced4",
+      "value": "Upload User Files:\t"
+     }
+    },
+    "1b7ee0de15484cd5aecd6d8ca3b6ee9d": {
+     "model_module": "@jupyter-widgets/controls",
+     "model_module_version": "1.5.0",
+     "model_name": "VBoxModel",
+     "state": {
+      "_dom_classes": [],
+      "_model_module": "@jupyter-widgets/controls",
+      "_model_module_version": "1.5.0",
+      "_model_name": "VBoxModel",
+      "_view_count": null,
+      "_view_module": "@jupyter-widgets/controls",
+      "_view_module_version": "1.5.0",
+      "_view_name": "VBoxView",
+      "box_style": "",
+      "children": [
+       "IPY_MODEL_b051a90758434644955747bc02d00bab",
+       "IPY_MODEL_dde20647d3594d31b66b19659f53a95e",
+       "IPY_MODEL_8610fffd2d2a4ec28f8c874c06073ce7",
+       "IPY_MODEL_654ab6d155eb457ea5c719a9ac27ad5b",
+       "IPY_MODEL_86cb4f568f454ff8832face502fb0745"
+      ],
+      "layout": "IPY_MODEL_e30fe87f01bc4580a61713b5b72439a2"
+     }
+    },
+    "252b8009f3734ed2908049ebb40c0247": {
+     "model_module": "@jupyter-widgets/controls",
+     "model_module_version": "1.5.0",
+     "model_name": "TextareaModel",
+     "state": {
+      "_dom_classes": [],
+      "_model_module": "@jupyter-widgets/controls",
+      "_model_module_version": "1.5.0",
+      "_model_name": "TextareaModel",
+      "_view_count": null,
+      "_view_module": "@jupyter-widgets/controls",
+      "_view_module_version": "1.5.0",
+      "_view_name": "TextareaView",
+      "continuous_update": true,
+      "description": "Learning Objectives",
+      "description_tooltip": null,
+      "disabled": false,
+      "layout": "IPY_MODEL_b74cf92175374028948d4cf529d4d1e6",
+      "placeholder": "Learning objectives: 1. Understand and implement classes in object-oriented programming",
+      "rows": null,
+      "style": "IPY_MODEL_f7d75b0a32554a9589c513336fc30095",
+      "value": ""
+     }
+    },
+    "26d13984d45745858d3b890bc7f18a90": {
+     "model_module": "@jupyter-widgets/controls",
+     "model_module_version": "1.5.0",
+     "model_name": "ButtonStyleModel",
+     "state": {
+      "_model_module": "@jupyter-widgets/controls",
+      "_model_module_version": "1.5.0",
+      "_model_name": "ButtonStyleModel",
+      "_view_count": null,
+      "_view_module": "@jupyter-widgets/base",
+      "_view_module_version": "1.2.0",
+      "_view_name": "StyleView",
+      "button_color": null,
+      "font_weight": ""
+     }
+    },
+    "453b12da4b6540cd9e4e57f73a4d670c": {
+     "model_module": "@jupyter-widgets/controls",
+     "model_module_version": "1.5.0",
+     "model_name": "DescriptionStyleModel",
+     "state": {
+      "_model_module": "@jupyter-widgets/controls",
+      "_model_module_version": "1.5.0",
+      "_model_name": "DescriptionStyleModel",
+      "_view_count": null,
+      "_view_module": "@jupyter-widgets/base",
+      "_view_module_version": "1.2.0",
+      "_view_name": "StyleView",
+      "description_width": ""
+     }
+    },
+    "49f80567705147f0b82d45b7f06dd1ba": {
+     "model_module": "@jupyter-widgets/controls",
+     "model_module_version": "1.5.0",
+     "model_name": "FileUploadModel",
+     "state": {
+      "_counter": 1,
+      "_dom_classes": [],
+      "_model_module": "@jupyter-widgets/controls",
+      "_model_module_version": "1.5.0",
+      "_model_name": "FileUploadModel",
+      "_view_count": null,
+      "_view_module": "@jupyter-widgets/controls",
+      "_view_module_version": "1.5.0",
+      "_view_name": "FileUploadView",
+      "accept": ".zip",
+      "button_style": "",
+      "data": [
+       null
+      ],
+      "description": "Upload",
+      "description_tooltip": null,
+      "disabled": false,
+      "error": "",
+      "icon": "upload",
+      "layout": "IPY_MODEL_dfa8d6c7d70b42468cbda035de89404c",
+      "metadata": [
+       {
+        "lastModified": 1689919477171,
+        "name": "instructorTest.zip",
+        "size": 4958,
+        "type": "application/zip"
+       }
+      ],
+      "multiple": false,
+      "style": "IPY_MODEL_26d13984d45745858d3b890bc7f18a90"
+     }
+    },
+    "53722998fbe64a7c94829b79e8cd69d6": {
+     "model_module": "@jupyter-widgets/base",
+     "model_module_version": "1.2.0",
+     "model_name": "LayoutModel",
+     "state": {
+      "_model_module": "@jupyter-widgets/base",
+      "_model_module_version": "1.2.0",
+      "_model_name": "LayoutModel",
+      "_view_count": null,
+      "_view_module": "@jupyter-widgets/base",
+      "_view_module_version": "1.2.0",
+      "_view_name": "LayoutView",
+      "align_content": null,
+      "align_items": null,
+      "align_self": null,
+      "border": null,
+      "bottom": null,
+      "display": null,
+      "flex": null,
+      "flex_flow": null,
+      "grid_area": null,
+      "grid_auto_columns": null,
+      "grid_auto_flow": null,
+      "grid_auto_rows": null,
+      "grid_column": null,
+      "grid_gap": null,
+      "grid_row": null,
+      "grid_template_areas": null,
+      "grid_template_columns": null,
+      "grid_template_rows": null,
+      "height": null,
+      "justify_content": null,
+      "justify_items": null,
+      "left": null,
+      "margin": null,
+      "max_height": null,
+      "max_width": null,
+      "min_height": null,
+      "min_width": null,
+      "object_fit": null,
+      "object_position": null,
+      "order": null,
+      "overflow": null,
+      "overflow_x": null,
+      "overflow_y": null,
+      "padding": null,
+      "right": null,
+      "top": null,
+      "visibility": null,
+      "width": null
+     }
+    },
+    "54e3918921f44fb4a9020beab951fcdf": {
+     "model_module": "@jupyter-widgets/controls",
+     "model_module_version": "1.5.0",
+     "model_name": "DescriptionStyleModel",
+     "state": {
+      "_model_module": "@jupyter-widgets/controls",
+      "_model_module_version": "1.5.0",
+      "_model_name": "DescriptionStyleModel",
+      "_view_count": null,
+      "_view_module": "@jupyter-widgets/base",
+      "_view_module_version": "1.2.0",
+      "_view_name": "StyleView",
+      "description_width": "initial"
+     }
+    },
+    "5a17f4509d194105b23dd616e45183d5": {
+     "model_module": "@jupyter-widgets/base",
+     "model_module_version": "1.2.0",
+     "model_name": "LayoutModel",
+     "state": {
+      "_model_module": "@jupyter-widgets/base",
+      "_model_module_version": "1.2.0",
+      "_model_name": "LayoutModel",
+      "_view_count": null,
+      "_view_module": "@jupyter-widgets/base",
+      "_view_module_version": "1.2.0",
+      "_view_name": "LayoutView",
+      "align_content": null,
+      "align_items": null,
+      "align_self": null,
+      "border": null,
+      "bottom": null,
+      "display": null,
+      "flex": null,
+      "flex_flow": null,
+      "grid_area": null,
+      "grid_auto_columns": null,
+      "grid_auto_flow": null,
+      "grid_auto_rows": null,
+      "grid_column": null,
+      "grid_gap": null,
+      "grid_row": null,
+      "grid_template_areas": null,
+      "grid_template_columns": null,
+      "grid_template_rows": null,
+      "height": null,
+      "justify_content": null,
+      "justify_items": null,
+      "left": null,
+      "margin": null,
+      "max_height": null,
+      "max_width": null,
+      "min_height": null,
+      "min_width": null,
+      "object_fit": null,
+      "object_position": null,
+      "order": null,
+      "overflow": null,
+      "overflow_x": null,
+      "overflow_y": null,
+      "padding": null,
+      "right": null,
+      "top": null,
+      "visibility": null,
+      "width": null
+     }
+    },
+    "60b80d550efa403a825a3cb913c26f53": {
+     "model_module": "@jupyter-widgets/base",
+     "model_module_version": "1.2.0",
+     "model_name": "LayoutModel",
+     "state": {
+      "_model_module": "@jupyter-widgets/base",
+      "_model_module_version": "1.2.0",
+      "_model_name": "LayoutModel",
+      "_view_count": null,
+      "_view_module": "@jupyter-widgets/base",
+      "_view_module_version": "1.2.0",
+      "_view_name": "LayoutView",
+      "align_content": null,
+      "align_items": null,
+      "align_self": null,
+      "border": null,
+      "bottom": null,
+      "display": null,
+      "flex": null,
+      "flex_flow": null,
+      "grid_area": null,
+      "grid_auto_columns": null,
+      "grid_auto_flow": null,
+      "grid_auto_rows": null,
+      "grid_column": null,
+      "grid_gap": null,
+      "grid_row": null,
+      "grid_template_areas": null,
+      "grid_template_columns": null,
+      "grid_template_rows": null,
+      "height": null,
+      "justify_content": null,
+      "justify_items": null,
+      "left": null,
+      "margin": null,
+      "max_height": null,
+      "max_width": null,
+      "min_height": null,
+      "min_width": null,
+      "object_fit": null,
+      "object_position": null,
+      "order": null,
+      "overflow": null,
+      "overflow_x": null,
+      "overflow_y": null,
+      "padding": null,
+      "right": null,
+      "top": null,
+      "visibility": null,
+      "width": null
+     }
+    },
+    "654ab6d155eb457ea5c719a9ac27ad5b": {
+     "model_module": "@jupyter-widgets/controls",
+     "model_module_version": "1.5.0",
+     "model_name": "ButtonModel",
+     "state": {
+      "_dom_classes": [],
+      "_model_module": "@jupyter-widgets/controls",
+      "_model_module_version": "1.5.0",
+      "_model_name": "ButtonModel",
+      "_view_count": null,
+      "_view_module": "@jupyter-widgets/controls",
+      "_view_module_version": "1.5.0",
+      "_view_name": "ButtonView",
+      "button_style": "success",
+      "description": "Submit",
+      "disabled": false,
+      "icon": "check",
+      "layout": "IPY_MODEL_81c4dda35a7d4e15821bb4bc0973354e",
+      "style": "IPY_MODEL_df1c46361f714aceb9c046f98fede40c",
+      "tooltip": ""
+     }
+    },
+    "6622f76f91f44527a87a7575bbd388d2": {
+     "model_module": "@jupyter-widgets/controls",
+     "model_module_version": "1.5.0",
+     "model_name": "HBoxModel",
+     "state": {
+      "_dom_classes": [],
+      "_model_module": "@jupyter-widgets/controls",
+      "_model_module_version": "1.5.0",
+      "_model_name": "HBoxModel",
+      "_view_count": null,
+      "_view_module": "@jupyter-widgets/controls",
+      "_view_module_version": "1.5.0",
+      "_view_name": "HBoxView",
+      "box_style": "",
+      "children": [
+       "IPY_MODEL_7f7164e80a464ba9b99f96c10132db25",
+       "IPY_MODEL_49f80567705147f0b82d45b7f06dd1ba"
+      ],
+      "layout": "IPY_MODEL_5a17f4509d194105b23dd616e45183d5"
+     }
+    },
+    "67b4083cd4234f52bb7cca27ab9cddb3": {
+     "model_module": "@jupyter-widgets/controls",
+     "model_module_version": "1.5.0",
+     "model_name": "FileUploadModel",
+     "state": {
+      "_counter": 0,
+      "_dom_classes": [],
+      "_model_module": "@jupyter-widgets/controls",
+      "_model_module_version": "1.5.0",
+      "_model_name": "FileUploadModel",
+      "_view_count": null,
+      "_view_module": "@jupyter-widgets/controls",
+      "_view_module_version": "1.5.0",
+      "_view_name": "FileUploadView",
+      "accept": ".zip",
+      "button_style": "",
+      "data": [],
+      "description": "Upload",
+      "description_tooltip": null,
+      "disabled": false,
+      "error": "",
+      "icon": "upload",
+      "layout": "IPY_MODEL_76548751bb9c4bcb9d4f39788ea7d4af",
+      "metadata": [],
+      "multiple": false,
+      "style": "IPY_MODEL_dbb88901f5084d49af208b91b52b6073"
+     }
+    },
+    "76548751bb9c4bcb9d4f39788ea7d4af": {
+     "model_module": "@jupyter-widgets/base",
+     "model_module_version": "1.2.0",
+     "model_name": "LayoutModel",
+     "state": {
+      "_model_module": "@jupyter-widgets/base",
+      "_model_module_version": "1.2.0",
+      "_model_name": "LayoutModel",
+      "_view_count": null,
+      "_view_module": "@jupyter-widgets/base",
+      "_view_module_version": "1.2.0",
+      "_view_name": "LayoutView",
+      "align_content": null,
+      "align_items": null,
+      "align_self": null,
+      "border": null,
+      "bottom": null,
+      "display": null,
+      "flex": null,
+      "flex_flow": null,
+      "grid_area": null,
+      "grid_auto_columns": null,
+      "grid_auto_flow": null,
+      "grid_auto_rows": null,
+      "grid_column": null,
+      "grid_gap": null,
+      "grid_row": null,
+      "grid_template_areas": null,
+      "grid_template_columns": null,
+      "grid_template_rows": null,
+      "height": null,
+      "justify_content": null,
+      "justify_items": null,
+      "left": null,
+      "margin": null,
+      "max_height": null,
+      "max_width": null,
+      "min_height": null,
+      "min_width": null,
+      "object_fit": null,
+      "object_position": null,
+      "order": null,
+      "overflow": null,
+      "overflow_x": null,
+      "overflow_y": null,
+      "padding": null,
+      "right": null,
+      "top": null,
+      "visibility": null,
+      "width": null
+     }
+    },
+    "7f7164e80a464ba9b99f96c10132db25": {
+     "model_module": "@jupyter-widgets/controls",
+     "model_module_version": "1.5.0",
+     "model_name": "LabelModel",
+     "state": {
+      "_dom_classes": [],
+      "_model_module": "@jupyter-widgets/controls",
+      "_model_module_version": "1.5.0",
+      "_model_name": "LabelModel",
+      "_view_count": null,
+      "_view_module": "@jupyter-widgets/controls",
+      "_view_module_version": "1.5.0",
+      "_view_name": "LabelView",
+      "description": "",
+      "description_tooltip": null,
+      "layout": "IPY_MODEL_60b80d550efa403a825a3cb913c26f53",
+      "placeholder": "​",
+      "style": "IPY_MODEL_d0bd0e3f12594ff1a51365b65a3fcc43",
+      "value": "Upload User Files:\t"
+     }
+    },
+    "7f814595d31e4b86992b5bd6bc85ced4": {
+     "model_module": "@jupyter-widgets/controls",
+     "model_module_version": "1.5.0",
+     "model_name": "DescriptionStyleModel",
+     "state": {
+      "_model_module": "@jupyter-widgets/controls",
+      "_model_module_version": "1.5.0",
+      "_model_name": "DescriptionStyleModel",
+      "_view_count": null,
+      "_view_module": "@jupyter-widgets/base",
+      "_view_module_version": "1.2.0",
+      "_view_name": "StyleView",
+      "description_width": ""
+     }
+    },
+    "81c4dda35a7d4e15821bb4bc0973354e": {
+     "model_module": "@jupyter-widgets/base",
+     "model_module_version": "1.2.0",
+     "model_name": "LayoutModel",
+     "state": {
+      "_model_module": "@jupyter-widgets/base",
+      "_model_module_version": "1.2.0",
+      "_model_name": "LayoutModel",
+      "_view_count": null,
+      "_view_module": "@jupyter-widgets/base",
+      "_view_module_version": "1.2.0",
+      "_view_name": "LayoutView",
+      "align_content": null,
+      "align_items": null,
+      "align_self": null,
+      "border": null,
+      "bottom": null,
+      "display": null,
+      "flex": null,
+      "flex_flow": null,
+      "grid_area": null,
+      "grid_auto_columns": null,
+      "grid_auto_flow": null,
+      "grid_auto_rows": null,
+      "grid_column": null,
+      "grid_gap": null,
+      "grid_row": null,
+      "grid_template_areas": null,
+      "grid_template_columns": null,
+      "grid_template_rows": null,
+      "height": null,
+      "justify_content": null,
+      "justify_items": null,
+      "left": null,
+      "margin": null,
+      "max_height": null,
+      "max_width": null,
+      "min_height": null,
+      "min_width": null,
+      "object_fit": null,
+      "object_position": null,
+      "order": null,
+      "overflow": null,
+      "overflow_x": null,
+      "overflow_y": null,
+      "padding": null,
+      "right": null,
+      "top": null,
+      "visibility": null,
+      "width": null
+     }
+    },
+    "8610fffd2d2a4ec28f8c874c06073ce7": {
+     "model_module": "@jupyter-widgets/controls",
+     "model_module_version": "1.5.0",
+     "model_name": "HBoxModel",
+     "state": {
+      "_dom_classes": [],
+      "_model_module": "@jupyter-widgets/controls",
+      "_model_module_version": "1.5.0",
+      "_model_name": "HBoxModel",
+      "_view_count": null,
+      "_view_module": "@jupyter-widgets/controls",
+      "_view_module_version": "1.5.0",
+      "_view_name": "HBoxView",
+      "box_style": "",
+      "children": [
+       "IPY_MODEL_1072a8a142f64dfd96ee528a2e9d1595",
+       "IPY_MODEL_67b4083cd4234f52bb7cca27ab9cddb3"
+      ],
+      "layout": "IPY_MODEL_d0a1ebdf7fc0473f91c39b29ca580934"
+     }
+    },
+    "86cb4f568f454ff8832face502fb0745": {
+     "model_module": "@jupyter-widgets/output",
+     "model_module_version": "1.0.0",
+     "model_name": "OutputModel",
+     "state": {
+      "_dom_classes": [],
+      "_model_module": "@jupyter-widgets/output",
+      "_model_module_version": "1.0.0",
+      "_model_name": "OutputModel",
+      "_view_count": null,
+      "_view_module": "@jupyter-widgets/output",
+      "_view_module_version": "1.0.0",
+      "_view_name": "OutputView",
+      "layout": "IPY_MODEL_53722998fbe64a7c94829b79e8cd69d6",
+      "msg_id": "",
+      "outputs": [
+       {
+        "name": "stdout",
+        "output_type": "stream",
+        "text": [
+         "Extracted files and directories: instructorTest/, __MACOSX/._instructorTest, instructorTest/bell_charreau.json, __MACOSX/instructorTest/._bell_charreau.json, instructorTest/spencer-smith_jesse.json, __MACOSX/instructorTest/._spencer-smith_jesse.json\n",
+         "\n",
+         "Loading successful!\n",
+         "Learning Objectives: \n",
+         "Extracted JSON files: instructorTest/spencer-smith_jesse.json, instructorTest/bell_charreau.json\n",
+         "Submitted and Reset all values.\n"
+        ]
+       }
+      ]
+     }
+    },
+    "a84d31fb8f4e4bafb74035158834b404": {
+     "model_module": "@jupyter-widgets/controls",
+     "model_module_version": "1.5.0",
+     "model_name": "VBoxModel",
+     "state": {
+      "_dom_classes": [],
+      "_model_module": "@jupyter-widgets/controls",
+      "_model_module_version": "1.5.0",
+      "_model_name": "VBoxModel",
+      "_view_count": null,
+      "_view_module": "@jupyter-widgets/controls",
+      "_view_module_version": "1.5.0",
+      "_view_name": "VBoxView",
+      "box_style": "",
+      "children": [
+       "IPY_MODEL_b051a90758434644955747bc02d00bab",
+       "IPY_MODEL_252b8009f3734ed2908049ebb40c0247",
+       "IPY_MODEL_6622f76f91f44527a87a7575bbd388d2",
+       "IPY_MODEL_654ab6d155eb457ea5c719a9ac27ad5b",
+       "IPY_MODEL_86cb4f568f454ff8832face502fb0745"
+      ],
+      "layout": "IPY_MODEL_e30fe87f01bc4580a61713b5b72439a2"
+     }
+    },
+    "abbecdc637694e7cb026e003244e7037": {
+     "model_module": "@jupyter-widgets/base",
+     "model_module_version": "1.2.0",
+     "model_name": "LayoutModel",
+     "state": {
+      "_model_module": "@jupyter-widgets/base",
+      "_model_module_version": "1.2.0",
+      "_model_name": "LayoutModel",
+      "_view_count": null,
+      "_view_module": "@jupyter-widgets/base",
+      "_view_module_version": "1.2.0",
+      "_view_name": "LayoutView",
+      "align_content": null,
+      "align_items": null,
+      "align_self": null,
+      "border": null,
+      "bottom": null,
+      "display": null,
+      "flex": null,
+      "flex_flow": null,
+      "grid_area": null,
+      "grid_auto_columns": null,
+      "grid_auto_flow": null,
+      "grid_auto_rows": null,
+      "grid_column": null,
+      "grid_gap": null,
+      "grid_row": null,
+      "grid_template_areas": null,
+      "grid_template_columns": null,
+      "grid_template_rows": null,
+      "height": null,
+      "justify_content": null,
+      "justify_items": null,
+      "left": null,
+      "margin": null,
+      "max_height": null,
+      "max_width": null,
+      "min_height": null,
+      "min_width": null,
+      "object_fit": null,
+      "object_position": null,
+      "order": null,
+      "overflow": null,
+      "overflow_x": null,
+      "overflow_y": null,
+      "padding": null,
+      "right": null,
+      "top": null,
+      "visibility": null,
+      "width": null
+     }
+    },
+    "b051a90758434644955747bc02d00bab": {
+     "model_module": "@jupyter-widgets/controls",
+     "model_module_version": "1.5.0",
+     "model_name": "HTMLModel",
+     "state": {
+      "_dom_classes": [],
+      "_model_module": "@jupyter-widgets/controls",
+      "_model_module_version": "1.5.0",
+      "_model_name": "HTMLModel",
+      "_view_count": null,
+      "_view_module": "@jupyter-widgets/controls",
+      "_view_module_version": "1.5.0",
+      "_view_name": "HTMLView",
+      "description": "",
+      "description_tooltip": null,
+      "layout": "IPY_MODEL_d16b25c7e9e948938c9303fbe8ae3dcc",
+      "placeholder": "​",
+      "style": "IPY_MODEL_453b12da4b6540cd9e4e57f73a4d670c",
+      "value": "<h2>Instructor Grading Configuration</h2>"
+     }
+    },
+    "b74cf92175374028948d4cf529d4d1e6": {
+     "model_module": "@jupyter-widgets/base",
+     "model_module_version": "1.2.0",
+     "model_name": "LayoutModel",
+     "state": {
+      "_model_module": "@jupyter-widgets/base",
+      "_model_module_version": "1.2.0",
+      "_model_name": "LayoutModel",
+      "_view_count": null,
+      "_view_module": "@jupyter-widgets/base",
+      "_view_module_version": "1.2.0",
+      "_view_name": "LayoutView",
+      "align_content": null,
+      "align_items": null,
+      "align_self": null,
+      "border": null,
+      "bottom": null,
+      "display": null,
+      "flex": null,
+      "flex_flow": null,
+      "grid_area": null,
+      "grid_auto_columns": null,
+      "grid_auto_flow": null,
+      "grid_auto_rows": null,
+      "grid_column": null,
+      "grid_gap": null,
+      "grid_row": null,
+      "grid_template_areas": null,
+      "grid_template_columns": null,
+      "grid_template_rows": null,
+      "height": null,
+      "justify_content": null,
+      "justify_items": null,
+      "left": null,
+      "margin": null,
+      "max_height": null,
+      "max_width": null,
+      "min_height": null,
+      "min_width": null,
+      "object_fit": null,
+      "object_position": null,
+      "order": null,
+      "overflow": null,
+      "overflow_x": null,
+      "overflow_y": null,
+      "padding": null,
+      "right": null,
+      "top": null,
+      "visibility": null,
+      "width": "auto"
+     }
+    },
+    "d0a1ebdf7fc0473f91c39b29ca580934": {
+     "model_module": "@jupyter-widgets/base",
+     "model_module_version": "1.2.0",
+     "model_name": "LayoutModel",
+     "state": {
+      "_model_module": "@jupyter-widgets/base",
+      "_model_module_version": "1.2.0",
+      "_model_name": "LayoutModel",
+      "_view_count": null,
+      "_view_module": "@jupyter-widgets/base",
+      "_view_module_version": "1.2.0",
+      "_view_name": "LayoutView",
+      "align_content": null,
+      "align_items": null,
+      "align_self": null,
+      "border": null,
+      "bottom": null,
+      "display": null,
+      "flex": null,
+      "flex_flow": null,
+      "grid_area": null,
+      "grid_auto_columns": null,
+      "grid_auto_flow": null,
+      "grid_auto_rows": null,
+      "grid_column": null,
+      "grid_gap": null,
+      "grid_row": null,
+      "grid_template_areas": null,
+      "grid_template_columns": null,
+      "grid_template_rows": null,
+      "height": null,
+      "justify_content": null,
+      "justify_items": null,
+      "left": null,
+      "margin": null,
+      "max_height": null,
+      "max_width": null,
+      "min_height": null,
+      "min_width": null,
+      "object_fit": null,
+      "object_position": null,
+      "order": null,
+      "overflow": null,
+      "overflow_x": null,
+      "overflow_y": null,
+      "padding": null,
+      "right": null,
+      "top": null,
+      "visibility": null,
+      "width": null
+     }
+    },
+    "d0bd0e3f12594ff1a51365b65a3fcc43": {
+     "model_module": "@jupyter-widgets/controls",
+     "model_module_version": "1.5.0",
+     "model_name": "DescriptionStyleModel",
+     "state": {
+      "_model_module": "@jupyter-widgets/controls",
+      "_model_module_version": "1.5.0",
+      "_model_name": "DescriptionStyleModel",
+      "_view_count": null,
+      "_view_module": "@jupyter-widgets/base",
+      "_view_module_version": "1.2.0",
+      "_view_name": "StyleView",
+      "description_width": ""
+     }
+    },
+    "d16b25c7e9e948938c9303fbe8ae3dcc": {
+     "model_module": "@jupyter-widgets/base",
+     "model_module_version": "1.2.0",
+     "model_name": "LayoutModel",
+     "state": {
+      "_model_module": "@jupyter-widgets/base",
+      "_model_module_version": "1.2.0",
+      "_model_name": "LayoutModel",
+      "_view_count": null,
+      "_view_module": "@jupyter-widgets/base",
+      "_view_module_version": "1.2.0",
+      "_view_name": "LayoutView",
+      "align_content": null,
+      "align_items": null,
+      "align_self": null,
+      "border": null,
+      "bottom": null,
+      "display": null,
+      "flex": null,
+      "flex_flow": null,
+      "grid_area": null,
+      "grid_auto_columns": null,
+      "grid_auto_flow": null,
+      "grid_auto_rows": null,
+      "grid_column": null,
+      "grid_gap": null,
+      "grid_row": null,
+      "grid_template_areas": null,
+      "grid_template_columns": null,
+      "grid_template_rows": null,
+      "height": null,
+      "justify_content": null,
+      "justify_items": null,
+      "left": null,
+      "margin": null,
+      "max_height": null,
+      "max_width": null,
+      "min_height": null,
+      "min_width": null,
+      "object_fit": null,
+      "object_position": null,
+      "order": null,
+      "overflow": null,
+      "overflow_x": null,
+      "overflow_y": null,
+      "padding": null,
+      "right": null,
+      "top": null,
+      "visibility": null,
+      "width": null
+     }
+    },
+    "dbb88901f5084d49af208b91b52b6073": {
+     "model_module": "@jupyter-widgets/controls",
+     "model_module_version": "1.5.0",
+     "model_name": "ButtonStyleModel",
+     "state": {
+      "_model_module": "@jupyter-widgets/controls",
+      "_model_module_version": "1.5.0",
+      "_model_name": "ButtonStyleModel",
+      "_view_count": null,
+      "_view_module": "@jupyter-widgets/base",
+      "_view_module_version": "1.2.0",
+      "_view_name": "StyleView",
+      "button_color": null,
+      "font_weight": ""
+     }
+    },
+    "dde20647d3594d31b66b19659f53a95e": {
+     "model_module": "@jupyter-widgets/controls",
+     "model_module_version": "1.5.0",
+     "model_name": "TextareaModel",
+     "state": {
+      "_dom_classes": [],
+      "_model_module": "@jupyter-widgets/controls",
+      "_model_module_version": "1.5.0",
+      "_model_name": "TextareaModel",
+      "_view_count": null,
+      "_view_module": "@jupyter-widgets/controls",
+      "_view_module_version": "1.5.0",
+      "_view_name": "TextareaView",
+      "continuous_update": true,
+      "description": "Learning Objectives",
+      "description_tooltip": null,
+      "disabled": false,
+      "layout": "IPY_MODEL_b74cf92175374028948d4cf529d4d1e6",
+      "placeholder": "Learning objectives: 1. Understand and implement classes in object-oriented programming",
+      "rows": null,
+      "style": "IPY_MODEL_54e3918921f44fb4a9020beab951fcdf",
+      "value": ""
+     }
+    },
+    "df1c46361f714aceb9c046f98fede40c": {
+     "model_module": "@jupyter-widgets/controls",
+     "model_module_version": "1.5.0",
+     "model_name": "ButtonStyleModel",
+     "state": {
+      "_model_module": "@jupyter-widgets/controls",
+      "_model_module_version": "1.5.0",
+      "_model_name": "ButtonStyleModel",
+      "_view_count": null,
+      "_view_module": "@jupyter-widgets/base",
+      "_view_module_version": "1.2.0",
+      "_view_name": "StyleView",
+      "button_color": null,
+      "font_weight": ""
+     }
+    },
+    "dfa8d6c7d70b42468cbda035de89404c": {
+     "model_module": "@jupyter-widgets/base",
+     "model_module_version": "1.2.0",
+     "model_name": "LayoutModel",
+     "state": {
+      "_model_module": "@jupyter-widgets/base",
+      "_model_module_version": "1.2.0",
+      "_model_name": "LayoutModel",
+      "_view_count": null,
+      "_view_module": "@jupyter-widgets/base",
+      "_view_module_version": "1.2.0",
+      "_view_name": "LayoutView",
+      "align_content": null,
+      "align_items": null,
+      "align_self": null,
+      "border": null,
+      "bottom": null,
+      "display": null,
+      "flex": null,
+      "flex_flow": null,
+      "grid_area": null,
+      "grid_auto_columns": null,
+      "grid_auto_flow": null,
+      "grid_auto_rows": null,
+      "grid_column": null,
+      "grid_gap": null,
+      "grid_row": null,
+      "grid_template_areas": null,
+      "grid_template_columns": null,
+      "grid_template_rows": null,
+      "height": null,
+      "justify_content": null,
+      "justify_items": null,
+      "left": null,
+      "margin": null,
+      "max_height": null,
+      "max_width": null,
+      "min_height": null,
+      "min_width": null,
+      "object_fit": null,
+      "object_position": null,
+      "order": null,
+      "overflow": null,
+      "overflow_x": null,
+      "overflow_y": null,
+      "padding": null,
+      "right": null,
+      "top": null,
+      "visibility": null,
+      "width": null
+     }
+    },
+    "e30fe87f01bc4580a61713b5b72439a2": {
+     "model_module": "@jupyter-widgets/base",
+     "model_module_version": "1.2.0",
+     "model_name": "LayoutModel",
+     "state": {
+      "_model_module": "@jupyter-widgets/base",
+      "_model_module_version": "1.2.0",
+      "_model_name": "LayoutModel",
+      "_view_count": null,
+      "_view_module": "@jupyter-widgets/base",
+      "_view_module_version": "1.2.0",
+      "_view_name": "LayoutView",
+      "align_content": null,
+      "align_items": "stretch",
+      "align_self": null,
+      "border": "solid 1px gray",
+      "bottom": null,
+      "display": "flex",
+      "flex": null,
+      "flex_flow": "column",
+      "grid_area": null,
+      "grid_auto_columns": null,
+      "grid_auto_flow": null,
+      "grid_auto_rows": null,
+      "grid_column": null,
+      "grid_gap": null,
+      "grid_row": null,
+      "grid_template_areas": null,
+      "grid_template_columns": null,
+      "grid_template_rows": null,
+      "height": null,
+      "justify_content": null,
+      "justify_items": null,
+      "left": null,
+      "margin": null,
+      "max_height": null,
+      "max_width": null,
+      "min_height": null,
+      "min_width": null,
+      "object_fit": null,
+      "object_position": null,
+      "order": null,
+      "overflow": null,
+      "overflow_x": null,
+      "overflow_y": null,
+      "padding": "0px 30px 20px 30px",
+      "right": null,
+      "top": null,
+      "visibility": null,
+      "width": "50%"
+     }
+    },
+    "f7d75b0a32554a9589c513336fc30095": {
+     "model_module": "@jupyter-widgets/controls",
+     "model_module_version": "1.5.0",
+     "model_name": "DescriptionStyleModel",
+     "state": {
+      "_model_module": "@jupyter-widgets/controls",
+      "_model_module_version": "1.5.0",
+      "_model_name": "DescriptionStyleModel",
+      "_view_count": null,
+      "_view_module": "@jupyter-widgets/base",
+      "_view_module_version": "1.2.0",
+      "_view_name": "StyleView",
+      "description_width": "initial"
+     }
+    }
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}

lo-achievement/instructor_intr_notebook_example_training.ipynb

@@ -0,0 +1,1277 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "view-in-github",
+        "colab_type": "text"
+      },
+      "source": [
+        "<a href=\"https://colab.research.google.com/github/vanderbilt-data-science/lo-achievement/blob/adding_grading_levels_to_instructor_nb/instructor_intr_notebook_example_training.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WMKrKfx8_3fc"
+      },
+      "source": [
+        "# Instructor Grading and Assessment\n",
+        "This notebook executes grading of student submissions based on the examples provided in the [Wiki](https://github.com/vanderbilt-data-science/lo-achievement/wiki/Examples-of-great,-good,-and-poor-answers-to-questions) from Dr. Jesse Blocher. In this iteration, we use the Unstructured File Loader, which cannot proccess .json files (the preferred format). We are working on finding a file loader that allows .json. In this version of the notebook, the model has only been trained on Question 2 from the notebook.\n",
+        "\n",
+        "To train the model, we used 2 out of the three student example from each grade brack and inputted into a .pdf with clearly defined levels. Then, we used the excluded answers to test the accuracy of the model's grading."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "# Load and Install Neccessary Libraries"
+      ],
+      "metadata": {
+        "id": "2UQgQSoMx4My"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "! pip install -q langchain=='0.0.229' openai gradio numpy chromadb tiktoken unstructured pdf2image pydantic==\"1.10.8\" jq"
+      ],
+      "metadata": {
+        "id": "UJi1Oy0CyPHD"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# import necessary libraries here\n",
+        "from getpass import getpass\n",
+        "from langchain.llms import OpenAI as openai\n",
+        "from langchain.chat_models import ChatOpenAI\n",
+        "from langchain.prompts import PromptTemplate\n",
+        "from langchain.document_loaders import TextLoader\n",
+        "from langchain.indexes import VectorstoreIndexCreator\n",
+        "from langchain.text_splitter import CharacterTextSplitter\n",
+        "from langchain.embeddings import OpenAIEmbeddings\n",
+        "from langchain.schema import SystemMessage, HumanMessage, AIMessage\n",
+        "import numpy as np\n",
+        "import os\n",
+        "from langchain.vectorstores import Chroma\n",
+        "from langchain.document_loaders.unstructured import UnstructuredFileLoader\n",
+        "from langchain.document_loaders import UnstructuredFileLoader\n",
+        "from langchain.chains import VectorDBQA\n",
+        "from langchain.document_loaders import JSONLoader\n",
+        "import json\n",
+        "from pathlib import Path\n",
+        "from pprint import pprint\n",
+        "\n",
+        "\n",
+        "from langchain.prompts.few_shot import FewShotPromptTemplate\n",
+        "from langchain.prompts.prompt import PromptTemplate"
+      ],
+      "metadata": {
+        "id": "YHytCUoExrYe"
+      },
+      "execution_count": 2,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "# Set up model and pass OpenAI Key\n",
+        "Here we are setting up the model and using a system message to pass a persona prompt with the grading advice"
+      ],
+      "metadata": {
+        "id": "4elyN72szz-_"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# setup open AI api key\n",
+        "openai_api_key = getpass()"
+      ],
+      "metadata": {
+        "id": "jVPEFX3ixJnM"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "os.environ[\"OPENAI_API_KEY\"] = openai_api_key\n",
+        "openai.api_key = openai_api_key"
+      ],
+      "metadata": {
+        "id": "obplpeB78h_M"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# Initiate model (model type and specify persona)\n",
+        "llm = ChatOpenAI(model='gpt-3.5-turbo-16k')\n",
+        "messages = [\n",
+        "    SystemMessage(content=\"You are a helpful grading assistant. In grading the following questions, keep in mind the advice from the professor: one aspect of it was being specific. The poor answers have a lot of platitudes, the better answers give specific examples. Secondly, they should discuss automation and/or prediction specifically. Those are the things that ML does, it is not 'technology' broadly.\"),\n",
+        "    HumanMessage(content=\"\")\n",
+        "]"
+      ],
+      "metadata": {
+        "id": "e_ZavOnS8iuE"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "# Original route: Vector Stores from .json files\n",
+        "This section uses .json files and vector stores with system/human messaging and a persona prompt tailored to Dr. Blocher's grading philosophy for the Wiki examples (only includes section 2 at this time)."
+      ],
+      "metadata": {
+        "id": "KHS3E6PydN-2"
+      }
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## Grading based on A, B, and C-level answers from previous students to Question 2 from the [Wiki](https://github.com/vanderbilt-data-science/lo-achievement/wiki/Examples-of-great,-good,-and-poor-answers-to-questions):\n",
+        "\n",
+        "**Question 2:** Why is machine learning so important for businesses? Answer this question generally (i.e. such that it applies to many or at least most businesses)."
+      ],
+      "metadata": {
+        "id": "IYCBSD_8l7uu"
+      }
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "### Creating .json file from case examples (Question 2)\n",
+        "The purpose of this cell is to create a json file based on the previously submitted, graded work of students based on the case file provided by Dr. Blocher in the Wiki. So, here you could create your own file, or for quick demo purposes you can use the zip file in the next section heading."
+      ],
+      "metadata": {
+        "id": "TYlGEusr64kA"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "q2 = 'Question 2: Why is machine learning so important for businesses? Answer this question generally (i.e. such that it applies to many or at least most businesses).'"
+      ],
+      "metadata": {
+        "id": "DAIaKTRYxOlh"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# A-level answers\n",
+        "\n",
+        "q2_A_answer_1 = 'Machine learning is extremely important tool for businesses. It can be used in a variety of ways, but most importantly, it can be used to identify patterns within their data that might not otherwise be identified by human beings. For example, it can understand customer behaviors, optimize logistics, and expand efficiencies throughout the business. Machine learning does not get tired, meaning it can work as long as you want it to. It can sift through massive amounts of data, that no human being can look through in an efficient manner. Machine learning can be used as a tool to identify anomalies when something needs to be checked to save or gain money. The predictions that companies gain from machine learning are cheap, accurate, and automate. These machine learning algorithms can be brought to larger scales to encompass the whole business and its operations. It is important to note, Machine learning is just predictions. Predictions to understand important patterns that could make or break a company since they understand the patterns of their business more. It is an amazing tool, but should be used wisely and carefully because if not, it can expensive, useless, and straight up wrong.'\n",
+        "q2_A_answer_2 = 'Machine learning is important for most of the sectors in business. Overall, it gives the company of an overview about what would be the trend for their business industry, and analyze the customer behavior to help business segment their customers groups. Today, many companies have a vast amount of information generated by behavior, computer, events, people, and devices. This massive amount of data is difficult for human to handle, and even if human manages it, it is not profitable as human labor is expensive. Thanks to machine learning, companies can utilize their in-house or even third-party data to make something useful for their business. In medical analysis, for example, with human, it takes a very long time to find patterns in thousands of MRI scans. On the other hand, machines can detect patterns in seconds by entering data as long as the information is correctly labeled or trained properly. Another example would be segmenting customer group. In marketing department, the business could use unsupervised machine learning to cluster their customer segments to generate personalized contents that are relevant for each of individuals.'\n",
+        "\n",
+        "# List creation\n",
+        "\n",
+        "q2_A_answers_list = [q2_A_answer_1, q2_A_answer_2]\n"
+      ],
+      "metadata": {
+        "id": "yQT6aExSr1dP"
+      },
+      "execution_count": 9,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# B-level answers\n",
+        "\n",
+        "q2_B_answer_1 = 'Companies use ML models to improve different aspects of their business, like manufacturing, hiring, deployment, advertising, etc. The main goal is to improve productive and increase profitability of the company. The ML models are fed with company and externally available data to help the company optimize its departments and in turn become more financially successful/ productive. For example, using purchasing history, the company can predict who to advertise products to, to increase sales.'\n",
+        "q2_B_answer_2 = 'Machine learning allows business to have automated decision, scale, predictive analysis and performance. Machine learning also helps a business have a data strategy. This is how a firm uses data, data infrastructure, governance, etc. to accomplish its strategic goals and maintain/grow their competitive advantage within their industry.'\n",
+        "q2_B_answer_3 = 'The short answer is ML can help make decisions for businesses. To be clarified, ML does not make decisions for businesses. I mean it can, but people have not trusted ML enough yet and ML has not been that good to let it directly make business decisions. Business people only use ML to help themselves get intuitions of how decisions should be made and make predictions of results they might get based on their decisions. For example, if a business tries to launch a new product, it will use ML to test whether it will work or not on a small scale before it is introduced to a large scale. People called this step piloting. In this step, people collect data that is generated by using the pilot product and analyze their next move. They could tweak some features based on the feedback. If they think the data in their interest shows the product performs well, they might predict that this product will be successful when it is introduced on a large scale. Then, they will launch it.'\n",
+        "# List creation\n",
+        "\n",
+        "q2_B_answers_list = [q2_B_answer_1, q2_B_answer_2, q2_B_answer_3]"
+      ],
+      "metadata": {
+        "id": "KB1CmeRwtRvf"
+      },
+      "execution_count": 10,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# C-level answers\n",
+        "\n",
+        "q2_C_answer_1 = 'Machine learning powers many of the services we use today, such as the recommendation systems of Spotify and Netflix; search engines such as Google and Bing; social media such as TikTok and Instagram; voices such as Siri and Alexa, the list can go on. All these examples show that machine learning is already starting to play a pivotal role in today\"s data-rich world. Machines can help us sift through useful information that can lead to big breakthroughs, and we have seen the widespread use of this technology in various industries such as finance, healthcare, insurance, manufacturing, transformational change, etc.'\n",
+        "q2_C_answer_2 = 'As technology advanced, there are tons of new data generated and stored. All industries experienced this surge in data, including business. There is a huge amount of business data stored and waited in the database of each firm and they need solutions to utilize these data. Machine learning is a very promising approach for firms to puts these data in and output a meaning pattern or result that could help the firms with their existing work. This could turn into a working product or provide insights that could enhance the efficiency of the company’s workflow. With machine learning, a firm could either enter a new market with the new product or save time and effort with the meaningful insights. Achieving these possibilities with the data they already owned is a low effort but high reward action. This is the reason machine learning is valued by many businesses recently.'\n",
+        "\n",
+        "# List creation\n",
+        "\n",
+        "q2_C_answers_list = [q2_C_answer_1, q2_C_answer_2]"
+      ],
+      "metadata": {
+        "id": "3diAz43othjc"
+      },
+      "execution_count": 11,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "q2_Q_and_A = [\"Question:\", q2, \"A-level Answers\", q2_A_answers_list, \"B-level Answers\", q2_B_answers_list, \"C-level Answers\", q2_C_answers_list]"
+      ],
+      "metadata": {
+        "id": "oAnrMSU6u9do"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "import json\n",
+        "from google.colab import files\n",
+        "\n",
+        "def save_example_answers(examples, filename='wiki_ABC_Q2examples.json'):\n",
+        "  with open(filename, 'w') as file:\n",
+        "    json.dump(examples, file)\n",
+        "  files.download(filename)\n",
+        "\n",
+        "save_example_answers(q2_Q_and_A)"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 17
+        },
+        "id": "B16iYMEnri9s",
+        "outputId": "3e565b3d-804c-4b5e-acc8-efeb955c6c14"
+      },
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "<IPython.core.display.Javascript object>"
+            ],
+            "application/javascript": [
+              "\n",
+              "    async function download(id, filename, size) {\n",
+              "      if (!google.colab.kernel.accessAllowed) {\n",
+              "        return;\n",
+              "      }\n",
+              "      const div = document.createElement('div');\n",
+              "      const label = document.createElement('label');\n",
+              "      label.textContent = `Downloading \"${filename}\": `;\n",
+              "      div.appendChild(label);\n",
+              "      const progress = document.createElement('progress');\n",
+              "      progress.max = size;\n",
+              "      div.appendChild(progress);\n",
+              "      document.body.appendChild(div);\n",
+              "\n",
+              "      const buffers = [];\n",
+              "      let downloaded = 0;\n",
+              "\n",
+              "      const channel = await google.colab.kernel.comms.open(id);\n",
+              "      // Send a message to notify the kernel that we're ready.\n",
+              "      channel.send({})\n",
+              "\n",
+              "      for await (const message of channel.messages) {\n",
+              "        // Send a message to notify the kernel that we're ready.\n",
+              "        channel.send({})\n",
+              "        if (message.buffers) {\n",
+              "          for (const buffer of message.buffers) {\n",
+              "            buffers.push(buffer);\n",
+              "            downloaded += buffer.byteLength;\n",
+              "            progress.value = downloaded;\n",
+              "          }\n",
+              "        }\n",
+              "      }\n",
+              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
+              "      const a = document.createElement('a');\n",
+              "      a.href = window.URL.createObjectURL(blob);\n",
+              "      a.download = filename;\n",
+              "      div.appendChild(a);\n",
+              "      a.click();\n",
+              "      div.remove();\n",
+              "    }\n",
+              "  "
+            ]
+          },
+          "metadata": {}
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "<IPython.core.display.Javascript object>"
+            ],
+            "application/javascript": [
+              "download(\"download_8ec6f24c-9653-4335-8c1b-766926434399\", \"wiki_ABC_Q2examples.json\", 5931)"
+            ]
+          },
+          "metadata": {}
+        }
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "### Creating a Vector Store\n",
+        "Here we create a vector store based on the .json file (which you can find at this [link](https://drive.google.com/file/d/1nk6JhbqoUHFie-Ewb436pdV-onlObZUt/view?usp=sharing), but will need to unzip)."
+      ],
+      "metadata": {
+        "id": "exXR-A2oxWeg"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# Upload .json or zip\n",
+        "from google.colab import files\n",
+        "uploaded = files.upload()"
+      ],
+      "metadata": {
+        "id": "Wpt6qsmEw8WP"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# Unzip file if neccessary\n",
+        "!unzip '/content/wiki_ABC_Q2examples (2).json.zip'"
+      ],
+      "metadata": {
+        "id": "7T2LpkiZh9LT"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# Create file path\n",
+        "data = '/content/wiki_ABC_Q2examples (2).json'"
+      ],
+      "metadata": {
+        "id": "CVY0CVvhxyCu"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# Load the .json\n",
+        "data = json.loads(Path(file_path).read_text())\n",
+        "data = str(data)"
+      ],
+      "metadata": {
+        "id": "dHjK0nN6yYPH"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# Create Vector Store\n",
+        "\n",
+        "text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)\n",
+        "texts = text_splitter.split_text(data)\n",
+        "\n",
+        "embeddings = OpenAIEmbeddings()\n",
+        "\n",
+        "db = Chroma.from_texts(texts, embeddings)\n",
+        "\n",
+        "qa = VectorDBQA.from_chain_type(llm=llm, chain_type=\"stuff\", vectorstore=db, k=1)"
+      ],
+      "metadata": {
+        "id": "paibxyeuxnu1"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# Creating examples from student answers to test the Wiki\n",
+        "\n",
+        "example1 = 'The most powerful aspect of machine learning is its ability to automate processes. If the goal is well defined and repeatable, an algorithm can be trained to perform that task far faster than any human and often with more reliability. Because of this, businesses can implement machine learning algorithms to solve problems human labor was previously required for, whether that be mental or physical labor. Although implementing machine learning often has high initial costs, because computers do not require payment outside of maintaining their operation, in the long run companies can save money by either making their workers’ tasks more efficient or by entirely automating tasks. This increases the profit margins for those firms.'\n",
+        "# B level\n",
+        "example2 ='ML systems can help improve access to data while managing compliance. It helps businesspeople deal with data in a more efficient way, since there is numerous data generated per second in such industry. If they use some of the simple classification models with high accuracy, it will save them a lot of time on data cleaning and validation, which are kind of repetitive and time-consuming. For example, NLP provides an easy way for personnel to query business information, understand business processes, and discover new relationships between business data, ideas based on intuition and insight often emerge. Using models to do prediction helps people making wiser decision. Since models can handle way more data than human, newly collected data can feed in the model and get some predictive result as a reference to the decision makers. This is significant because in this industry, time is precious, and traders must decide quickly and precisely. A little negligence will lead to a big mistake, lose a lot of money, and even affect the company\"s reputation. Models can see patterns that are not easy for human to spot, which is also valuable for modify the way people doing analysis and interpret.'\n",
+        "# C level\n",
+        "example3 = 'The machine learning model (or one a broader view, artificial intelligence) is about prediction. According to the lecture, there are tree main innovations in it. Prediction is cheap, more accurate and automated. As a result, armed with machine learning, businesses could automatically gain much more accurate and powerful capabilities in forecasting, leading to a big savings both in time and money.'\n",
+        "\n",
+        "# Randomized list of answers\n",
+        "training_answers = [example2, example1, example3]"
+      ],
+      "metadata": {
+        "id": "-fyY-ftV-ZbG"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## Grading based on A, B, and C-level answers from previous students to Question 4 from the [Wiki](https://github.com/vanderbilt-data-science/lo-achievement/wiki/Examples-of-great,-good,-and-poor-answers-to-questions):\n",
+        "\n",
+        "**Question 4:** Describe an example where machine learning is not likely to work well. This should be an example where intelligent, rational people could think of using machine learning or even are already implementing it. Describe specifically why you think ML is unlikely to work well. This should not be an example where someone is implementing it badly (i.e. skipping testing). Rather, it should be an example of how ML simply will not work well, no matter how many smart people work carefully on the problem. You may use examples we’ve discussed in class; it does not need to be a novel application."
+      ],
+      "metadata": {
+        "id": "sSnxqnGDmPzC"
+      }
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "### Creating .json file from case examples (Question 4)\n",
+        "The purpose of this cell is to create a json file based on the previously submitted, graded work of students based on the case file provided by Dr. Blocher in the Wiki."
+      ],
+      "metadata": {
+        "id": "RMkTwaEFmhWd"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# Context: question\n",
+        "\n",
+        "q4 = 'Question 4: Describe an example where machine learning is not likely to work well. This should be an example where intelligent, rational people could think of using machine learning or even are already implementing it. Describe specifically why you think ML is unlikely to work well. This should not be an example where someone is implementing it badly (i.e. skipping testing). Rather, it should be an example of how ML simply will not work well, no matter how many smart people work carefully on the problem. You may use examples we’ve discussed in class; it does not need to be a novel application.'"
+      ],
+      "metadata": {
+        "id": "_UVEgvzKmltw"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# A-level answers\n",
+        "\n",
+        "q4_A_answer_1 = 'This is one of the cases discussed in class: Machine learning on the management of at-risk children. Traditionally, at-risk children are mostly determined by social workers and they would report to their agencies if some children are at-risk. The at-risk children will subsequentially be taken into custody before their circumstance is evaluated and a decision about their future is reached. If a machine-learning algorithm were to replace the aforementioned social worker: the biggest concern would be when the algorithm outputs a false positive or false negative prediction. A false positive means that the children are said to be at risk while actually not and removed from their households. A false negative would be when some children are not removed from the dangerous household because the algorithm returned negative results. The costs of both incorrect decisions are too big for anyone to afford. Admittedly, data scientists might be able to develop a great algorithm with very low false positive and false negative rates, and that social workers can easily make the same amount of mistakes or more, but it is still too much for our humane society to take by putting such a life-changing or even life-or-death decision in a machine’s metaphorical hands.'\n",
+        "q4_A_answer_2 = 'I believe machine learning will not work well in the court system when making sentences and decisions. I have heard in the news that machine learning was helping judges sentence defenders. Currently, I am unaware if this is still happening but this will not work because our court system has a racial basis. This racial bias stems back hundreds of years and a machine-learning model would exploit the discrimination already so present in our system. The dataset used to train the model has bias that would unfairly sentence a person because of their demographic. This overall is bad because there is no dataset you can use to train a model that already does not include some sort of inherited bias.'\n",
+        "\n",
+        "# List creation\n",
+        "\n",
+        "q4_A_answers_list = [q4_A_answer_1, q4_A_answer_2]\n"
+      ],
+      "metadata": {
+        "id": "Irw0U9PBmupJ"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# B-level answers\n",
+        "\n",
+        "q4_B_answer_1 = 'I believe that machine learning won’t work well in the advent of self-driving cars. As discussed in class, machines are bad at “analogies, small data or rare events, and identifying causality” (Lecture 4: Data Strategy.) While most driving occurrences are habitual and can be predicted, especially day-to-day events like driving to work or the grocery store, so much driving is small or rare events that can’t necessarily be predicted. For example, you can’t predict that the person behind you is texting while driving and is going to rear end you, or that a snowstorm can make you start to hydroplane on the road. So much of the process of learning is understanding the exceptions, such as driving in the rain, at night, or almost avoiding an accident. For self-driving cars to be successful, so much data is required to reduce error, which I don’t think is possible to acquire. While most algorithms can have leeway in their accuracy, self-driving cars cannot have any errors — a difficult feat.'\n",
+        "q4_B_answer_2 = 'As we discussed in class, in a hospital setting, ML cannot do the task of diagnosing medical conditions well. While ML have been used in some diagnostic tasks, such as detecting some certain types of cancer, it generally to be considered difficult and will not be used in a large scale because medical conditions can have many different conditions can have many different symptoms and can be difficult to diagnose even for experienced doctors. The training of a machine learning model for diagnosing medical conditions requires a large dataset of medical records, along with labels indicating each diagnosis. But there are several barriers. First, medical records are often private and sensitive, and sharing them may raise legal and ethical concerns. Second, medical conditions are complicated, it’s hard to ensure the labels are correct, some of which may takes years or decades to testify. Even if they can recognize some pattern for some disease, symptoms vary from person to person. At a first glance, even after running some tests, it’s hard for doctors to tell the case. It will be harder to use ML in diagnose in general.'\n",
+        "\n",
+        "q4_B_answers_list = [q4_B_answer_1, q4_B_answer_2]"
+      ],
+      "metadata": {
+        "id": "haKh_FsenmT3"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# C-level answers\n",
+        "\n",
+        "q4_C_answer_1 = 'One example would be unstable models. Some of the models frequently exhibit extreme instability and degrade over time. In such circumstances, the company can call for model monitoring and high-frequency model revision. Businesses may begin to revert to intuition-based approach when model creation lead times increase.'\n",
+        "q4_C_answer_2 = 'The example I want to describe is about GPS. We have heard that some drivers who blindly followed their GPS into disaster. The news reported that Three tourists from Bellevue, Washington, got lost after midnight in 2011 when they were unable to find their way back to the hotel. They took what they believed to be a road that would take them to the freeway after requesting the GPS to reroute them. Instead, their SUV submerged into a large body of water. These people trust algorithms and data more than their own logic and judgement. People who drive into the water just because they believe too much in GPS. Algorithms will never think like a human being and cannot make moral judgement like a human being. Machine learning is unable to provide any guidance on the accepted norms. Machine minds & human minds could never be the same. Everyone’s answer is all different, We cannot collect everyone’s information around the world, that is so hard. As long as there is one person’s information not counted, then it will have difference from what you predict. The world always require human’s judgement and ethics, that is the point can not be changed.'\n",
+        "q4_C_answer_3 = 'The organization and the abundance of your data is paramount in the applying machine learning. When your data are either incomplete, like not enough data, or unstructured, like messy data with a lot of missingness and inconsistencies, you are not likely to obtain a good result even from implementing the best model in the world. There are some techniques that you might be able to amplify your data or transfer other data into your own, such as transfer learning or domain adaptation. However, it is still strongly recommended that you should have plenty of data to perform machine learning on. Your data should also be able to well represent the question you are trying to answer. Low coverage of the true target population is likely to result in skewness of the result. Machines learning is not likely to work well in this scenario.'\n",
+        "\n",
+        "# List creation\n",
+        "\n",
+        "q4_C_answers_list = [q4_C_answer_1, q4_C_answer_2, q4_C_answer_3]"
+      ],
+      "metadata": {
+        "id": "YNHULek7oCE_"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "q4_Q_and_A = [\"Question:\", q4, \"A-level Answers\", q4_A_answers_list, \"B-level Answers\", q4_B_answers_list, \"C-level Answers\", q4_C_answers_list]"
+      ],
+      "metadata": {
+        "id": "9bXSVmPnojIh"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "import json\n",
+        "from google.colab import files\n",
+        "\n",
+        "def save_example_answers(examples, filename='wiki_ABC_Q4examples.json'):\n",
+        "  with open(filename, 'w') as file:\n",
+        "    json.dump(examples, file)\n",
+        "  files.download(filename)\n",
+        "\n",
+        "save_example_answers(q4_Q_and_A)"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 17
+        },
+        "outputId": "3e565b3d-804c-4b5e-acc8-efeb955c6c14",
+        "id": "Qqwk0mvroqWz"
+      },
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "<IPython.core.display.Javascript object>"
+            ],
+            "application/javascript": [
+              "\n",
+              "    async function download(id, filename, size) {\n",
+              "      if (!google.colab.kernel.accessAllowed) {\n",
+              "        return;\n",
+              "      }\n",
+              "      const div = document.createElement('div');\n",
+              "      const label = document.createElement('label');\n",
+              "      label.textContent = `Downloading \"${filename}\": `;\n",
+              "      div.appendChild(label);\n",
+              "      const progress = document.createElement('progress');\n",
+              "      progress.max = size;\n",
+              "      div.appendChild(progress);\n",
+              "      document.body.appendChild(div);\n",
+              "\n",
+              "      const buffers = [];\n",
+              "      let downloaded = 0;\n",
+              "\n",
+              "      const channel = await google.colab.kernel.comms.open(id);\n",
+              "      // Send a message to notify the kernel that we're ready.\n",
+              "      channel.send({})\n",
+              "\n",
+              "      for await (const message of channel.messages) {\n",
+              "        // Send a message to notify the kernel that we're ready.\n",
+              "        channel.send({})\n",
+              "        if (message.buffers) {\n",
+              "          for (const buffer of message.buffers) {\n",
+              "            buffers.push(buffer);\n",
+              "            downloaded += buffer.byteLength;\n",
+              "            progress.value = downloaded;\n",
+              "          }\n",
+              "        }\n",
+              "      }\n",
+              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
+              "      const a = document.createElement('a');\n",
+              "      a.href = window.URL.createObjectURL(blob);\n",
+              "      a.download = filename;\n",
+              "      div.appendChild(a);\n",
+              "      a.click();\n",
+              "      div.remove();\n",
+              "    }\n",
+              "  "
+            ]
+          },
+          "metadata": {}
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "<IPython.core.display.Javascript object>"
+            ],
+            "application/javascript": [
+              "download(\"download_8ec6f24c-9653-4335-8c1b-766926434399\", \"wiki_ABC_Q2examples.json\", 5931)"
+            ]
+          },
+          "metadata": {}
+        }
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## Grading based on A, B, and C-level answers from previous students to Question 5 from the [Wiki](https://github.com/vanderbilt-data-science/lo-achievement/wiki/Examples-of-great,-good,-and-poor-answers-to-questions):\n",
+        "\n",
+        "**Question 5:** Describe a new example where you think machine learning could work. This cannot be examples we’ve given in class like customer churn, fraud detection, image or object recognition, disease diagnosis, stock prediction, self-driving cars, etc. t does not need to be completely novel, e.g., you can look for news stories and find something new. I want something that is different from the commonly used examples. You may use an example that you or another student brought up in class if it meets these criteria. Be sure to explain why you think it is an example where ML could work. Note: You do not need to be proven right that ML works. I am mostly interested in your logic and arguments of why you think it is a good idea, drawing on what we’ve discussed in class."
+      ],
+      "metadata": {
+        "id": "RfofiUreov7C"
+      }
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "### Creating .json file from case examples (Question 5)\n",
+        "The purpose of this cell is to create a json file based on the previously submitted, graded work of students based on the case file provided by Dr. Blocher in the Wiki."
+      ],
+      "metadata": {
+        "id": "P2h8XtXQo6eO"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# Context: question\n",
+        "\n",
+        "q5 = 'Question 5: Describe a new example where you think machine learning could work. This cannot be examples we’ve given in class like customer churn, fraud detection, image or object recognition, disease diagnosis, stock prediction, self-driving cars, etc. t does not need to be completely novel, e.g., you can look for news stories and find something new. I want something that is different from the commonly used examples. You may use an example that you or another student brought up in class if it meets these criteria. Be sure to explain why you think it is an example where ML could work. Note: You do not need to be proven right that ML works. I am mostly interested in your logic and arguments of why you think it is a good idea, drawing on what we’ve discussed in class.'"
+      ],
+      "metadata": {
+        "id": "evpoXMUSpCi4"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# A-level answers\n",
+        "\n",
+        "q5_A_answer_1 = 'As I recall, our group discussed that using ML to do intrusion detection for computers. Nowadays, people’s information is digitalized and stored in databases. Since databases are also servers (i.e., computers), there must be some vulnerability. Software is more common to do detection for now, but it can just recognize the intrusion pattern that already known or preset by the programmer. However, this can be accomplished by employing algorithms that can analyze large amounts of data and detect anomalies that may indicate an attempted intrusion. We can feed ML model with network traffic, labeled or not labeled, system logs to let it learn and identify unusual patterns or behavior. With well-learned models, we can use them to find out signs of unauthorized access or even malware. One of the greatest things is that ML model may be able to find some hidden pattern that human has not and predict how and when the intrusion will take place, or even figure out how the malware will function to attach. This trait is one of the most important traits that distinguish ML model from regular software.'\n",
+        "q5_A_answer_2 = 'Machine learning can be used in NLP (natural language processing). It is very useful to deal with massive unstructured text data. For example, we want to build a machine learning model to reflect the personality of Einstein according to the letters he wrote in the past decades to his family and friends. We can use machine learning models for NLP to transfer the unstructured text data to usable data first. Then we will use these data to train the model. After training the model by the letters Einstein wrote in the past decades, the model will have the same or similar personality as Einstein. And now we input the test letters into the model again, we expect the model will give us the sentiment scores of each letter (for example: humor-0.732, joyful-0.193, fear-0.015, sad-0.002). After we got the scores for each letter, we use statistical methods to analyze the scores and we will finally get an aggregate personality of Einstein.'\n",
+        "q5_A_answer_3 = 'The sports world is a place where I generally think machine learning can be well-utilized, but not in all cases. Something I’m thinking about is fantasy football, in which football fans can draft a team and decide who to “start” on their team for a given week and who to “sit.” NFL pundits and commentators can absolutely give ideas on who would be optimal, and do a great job at it, but when it comes to actual predictions for how many points a particular player might score in a given week, based on opponent average stats, expected workload, and other factors, while also measuring that against every other potential player, having an algorithm to back this up is extremely helpful. Sports betting can also benefit from these types of algorithms, and being able to accurately predict something as volatile as sports could have major business (and fandom) implications.'\n",
+        "\n",
+        "# List creation\n",
+        "\n",
+        "q5_A_answers_list = [q5_A_answer_1, q5_A_answer_2, q5_A_answer_3]\n"
+      ],
+      "metadata": {
+        "id": "B9abYhEupKA_"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# B-level answers\n",
+        "\n",
+        "q5_B_answer_1 = 'Machine learning can work well when it comes to finance organization, planning, and tracking. Since machine learning works well with statistical inference, it’s useful in decision making. For things such as planning and organizing, one example that machine learning can work well in is identifying price values for toiletries, groceries, and necessity items across various store within a specific location and output the best location for the user. We know that certain retailers mark up prices for the items on their shelfs. Some items are more expensive in one location than others and often time, certain prices are not posted online. In addition to that, traveling to different stores and compare prices are very inefficient because it can waste gas and time. To solve the problem, the algorithm will examine the user’s location and the item(s) that the user wants and output the best stores for the items. How this item work is that they take in data from all the closest stores, rank each item based on value, distance, and availability. On top of that, if users compile a lists of necessity items and output them in the model, the model can use data and compare stores that have more than one item on the list plus the other features (value, distance, and availability), and recommend the place, or places if item are not available, that could cut down time and expense for the users.'\n",
+        "q5_B_answer_2 = 'Natural gas consumption has rebounded in the last year, with economic recovery and increased extreme weather leading to increased demand for natural gas, resulting in a tight market and soaring prices. Gas supply agencies want to use ML to predict the deliverability of natural gas storage in depleted reservoirs. First, natural gas is most commonly stored in underground formations: 1) depleted reservoirs, 2) aquifers, and 3) salt caverns. A depleted reservoir must have some elements of a good oil and gas formation to be converted to a subsurface natural gas reservoir, such as good porosity and permeability, the presence of good seal rock, and the presence of cap rock. I think this thing can be achieved with ML because there is a huge amount of data collected from natural gas extraction sites around the world since 1915, and this data provides a good basis and realizability for our research. We can test and build models from this data so that we can use ML to make predictions.'\n",
+        "\n",
+        "# List creation\n",
+        "\n",
+        "q5_B_answers_list = [q5_B_answer_1, q5_B_answer_2]"
+      ],
+      "metadata": {
+        "id": "r0_Tt0lspkQ-"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# C-level answers\n",
+        "\n",
+        "q5_C_answer_1 = 'One example where ML would work in is speech recognition, converting speech into text.'\n",
+        "q5_C_answer_2 = 'ML may work well with education industry. Teachers can utilize ML for lesson preparation, such as analyze academic data, recommendation of high-quality teaching resources, on-demand generation of teaching plans. Others might be like intelligent teaching assistants instead of an actual person TA, or dynamically adjust the teaching content based on comprehensive factors such as learning situation analysis and learner style, and student feedback.'\n",
+        "\n",
+        "# List creation\n",
+        "\n",
+        "q5_C_answers_list = [q5_C_answer_1, q5_C_answer_2]"
+      ],
+      "metadata": {
+        "id": "oRlId1mtp46L"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "q5_Q_and_A = [\"Question:\", q5, \"A-level Answers\", q5_A_answers_list, \"B-level Answers\", q5_B_answers_list, \"C-level Answers\", q5_C_answers_list]"
+      ],
+      "metadata": {
+        "id": "XJC2UNxSqHBp"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "import json\n",
+        "from google.colab import files\n",
+        "\n",
+        "def save_example_answers(examples, filename='wiki_ABC_Q5examples.json'):\n",
+        "  with open(filename, 'w') as file:\n",
+        "    json.dump(examples, file)\n",
+        "  files.download(filename)\n",
+        "\n",
+        "save_example_answers(q5_Q_and_A)"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 17
+        },
+        "outputId": "3e565b3d-804c-4b5e-acc8-efeb955c6c14",
+        "id": "MRcW9_1MqPG-"
+      },
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "<IPython.core.display.Javascript object>"
+            ],
+            "application/javascript": [
+              "\n",
+              "    async function download(id, filename, size) {\n",
+              "      if (!google.colab.kernel.accessAllowed) {\n",
+              "        return;\n",
+              "      }\n",
+              "      const div = document.createElement('div');\n",
+              "      const label = document.createElement('label');\n",
+              "      label.textContent = `Downloading \"${filename}\": `;\n",
+              "      div.appendChild(label);\n",
+              "      const progress = document.createElement('progress');\n",
+              "      progress.max = size;\n",
+              "      div.appendChild(progress);\n",
+              "      document.body.appendChild(div);\n",
+              "\n",
+              "      const buffers = [];\n",
+              "      let downloaded = 0;\n",
+              "\n",
+              "      const channel = await google.colab.kernel.comms.open(id);\n",
+              "      // Send a message to notify the kernel that we're ready.\n",
+              "      channel.send({})\n",
+              "\n",
+              "      for await (const message of channel.messages) {\n",
+              "        // Send a message to notify the kernel that we're ready.\n",
+              "        channel.send({})\n",
+              "        if (message.buffers) {\n",
+              "          for (const buffer of message.buffers) {\n",
+              "            buffers.push(buffer);\n",
+              "            downloaded += buffer.byteLength;\n",
+              "            progress.value = downloaded;\n",
+              "          }\n",
+              "        }\n",
+              "      }\n",
+              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
+              "      const a = document.createElement('a');\n",
+              "      a.href = window.URL.createObjectURL(blob);\n",
+              "      a.download = filename;\n",
+              "      div.appendChild(a);\n",
+              "      a.click();\n",
+              "      div.remove();\n",
+              "    }\n",
+              "  "
+            ]
+          },
+          "metadata": {}
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "<IPython.core.display.Javascript object>"
+            ],
+            "application/javascript": [
+              "download(\"download_8ec6f24c-9653-4335-8c1b-766926434399\", \"wiki_ABC_Q2examples.json\", 5931)"
+            ]
+          },
+          "metadata": {}
+        }
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        " # Exploring with Langchain Example Sets, FewShotPrompts and Example Selectors\n",
+        " Here we're exploring another avenue for this goal to see if we can yield better results than the vector stores. Again, we are using question 2 from the Wiki to test."
+      ],
+      "metadata": {
+        "id": "PpKRr5Vw4_0F"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# Context: question\n",
+        "\n",
+        "q2 = 'Question 2: Why is machine learning so important for businesses? Answer this question generally (i.e. such that it applies to many or at least most businesses).'"
+      ],
+      "metadata": {
+        "id": "-kmMFUaLs_Q1"
+      },
+      "execution_count": 7,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from langchain.prompts.few_shot import FewShotPromptTemplate\n",
+        "from langchain.prompts.prompt import PromptTemplate\n",
+        "\n",
+        "examples = [\n",
+        "  {\n",
+        "    \"question\": f\"\"\" Please grade the following student's answer to the question ({q2})\n",
+        "     on an A-, B-, C-level grading scale: {q2_A_answer_1}.\"\"\",\n",
+        "    \"answer\": \"This student should recieve an A-level grade.\"\n",
+        "  },\n",
+        "  {\n",
+        "    \"question\": f\"\"\" Please grade the following student's answer to the question ({q2})\n",
+        "     on an A-, B-, C-level grading scale: {q2_A_answer_2}.\"\"\",\n",
+        "    \"answer\": \"The student should recieve an A-level grade.\"\n",
+        "  },\n",
+        "  {\n",
+        "    \"question\": f\"\"\" Please grade the following student's answer to the question ({q2})\n",
+        "     on an A-, B-, C-level grading scale: {q2_B_answer_1}.\"\"\",\n",
+        "    \"answer\": \"This student should recieve a B-level grade.\"\n",
+        "  },\n",
+        "  {\n",
+        "    \"question\": f\"\"\" Please grade the following student's answer to the question ({q2})\n",
+        "     on an A-, B-, C-level grading scale: {q2_B_answer_2}.\"\"\",\n",
+        "    \"answer\": \"This student should recieve a B-level grade.\"\n",
+        "  },\n",
+        "  {\n",
+        "    \"question\": f\"\"\" Please grade the following student's answer to the question ({q2})\n",
+        "     on an A-, B-, C-level grading scale: {q2_B_answer_3}.\"\"\",\n",
+        "    \"answer\": \"This student should recieve a B-level grade.\"\n",
+        "  },\n",
+        "  {\n",
+        "    \"question\": f\"\"\" Please grade the following student's answer to the question ({q2})\n",
+        "     on an A-, B-, C-level grading scale: {q2_C_answer_1}.\"\"\",\n",
+        "    \"answer\": \"This student should recieve a C-level grade.\"\n",
+        "  },\n",
+        "  {\n",
+        "    \"question\": f\"\"\" Please grade the following student's answer to the question ({q2})\n",
+        "     on an A-, B-, C-level grading scale: {q2_C_answer_2}.\"\"\",\n",
+        "    \"answer\": \"This student should recieve a C-level grade.\"\n",
+        "  }\n",
+        "]"
+      ],
+      "metadata": {
+        "id": "1idnQLYW-697"
+      },
+      "execution_count": 12,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# Example selector\n",
+        "from langchain.prompts.example_selector import SemanticSimilarityExampleSelector, MaxMarginalRelevanceExampleSelector, NGramOverlapExampleSelector\n",
+        "from langchain.vectorstores import Chroma\n",
+        "from langchain.embeddings import OpenAIEmbeddings\n",
+        "\n",
+        "\n",
+        "example_selector_semantic = SemanticSimilarityExampleSelector.from_examples(\n",
+        "    # This is the list of examples available to select from.\n",
+        "    examples,\n",
+        "    # This is the embedding class used to produce embeddings which are used to measure semantic similarity.\n",
+        "    OpenAIEmbeddings(),\n",
+        "    # This is the VectorStore class that is used to store the embeddings and do a similarity search over.\n",
+        "    Chroma,\n",
+        "    # This is the number of examples to produce.\n",
+        "    k=1\n",
+        ")\n",
+        "\n",
+        "example_selector_mmr = MaxMarginalRelevanceExampleSelector.from_examples(\n",
+        "    # This is the list of examples available to select from.\n",
+        "    examples,\n",
+        "    # This is the embedding class used to produce embeddings which are used to measure semantic similarity.\n",
+        "    OpenAIEmbeddings(),\n",
+        "    # This is the VectorStore class that is used to store the embeddings and do a similarity search over.\n",
+        "    Chroma,\n",
+        "    # This is the number of examples to produce.\n",
+        "    k=1,\n",
+        ")\n",
+        "\n",
+        "example_selector_ngram = NGramOverlapExampleSelector(\n",
+        "    # These are the examples it has available to choose from.\n",
+        "    examples=examples,\n",
+        "    # This is the PromptTemplate being used to format the examples.\n",
+        "    example_prompt=example_prompt,\n",
+        "    # This is the threshold, at which selector stops.\n",
+        "    # It is set to -1.0 by default.\n",
+        "    threshold=-1.0,\n",
+        "    # For negative threshold:\n",
+        "    # Selector sorts examples by ngram overlap score, and excludes none.\n",
+        "    # For threshold greater than 1.0:\n",
+        "    # Selector excludes all examples, and returns an empty list.\n",
+        "    # For threshold equal to 0.0:\n",
+        "    # Selector sorts examples by ngram overlap score,\n",
+        "    # and excludes those with no ngram overlap with input.\n",
+        ")\n",
+        "\n"
+      ],
+      "metadata": {
+        "id": "Y7vNiHKhuIjm"
+      },
+      "execution_count": 15,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "example_prompt = PromptTemplate(input_variables=[\"question\", \"answer\"], template=\"Question: {question}\\n{answer}\")\n",
+        "\n",
+        "print(example_prompt.format(**examples[0]))"
+      ],
+      "metadata": {
+        "id": "x2dCmQdACwGi"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "prompt = FewShotPromptTemplate(\n",
+        "    #examples=examples,\n",
+        "    example_selector = example_selector_semantic,\n",
+        "    example_prompt = example_prompt,\n",
+        "    suffix=\"Question: {input}\",\n",
+        "    input_variables=[\"input\"]\n",
+        ")\n",
+        "\n",
+        "print(prompt.format(input=f\"\"\" Please grade the following student's answer to the question ({q2})\n",
+        "     on an A-, B-, C-level grading scale: {example1}.\"\"\"))"
+      ],
+      "metadata": {
+        "id": "Uc_BvY0rBZgx"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "prompt_semantic = FewShotPromptTemplate(\n",
+        "    #examples=examples,\n",
+        "    example_selector = example_selector_semantic,\n",
+        "    example_prompt = example_prompt,\n",
+        "    suffix=\"Question: {input}\",\n",
+        "    input_variables=[\"input\"]\n",
+        ")\n",
+        "\n",
+        "prompt_mmr = FewShotPromptTemplate(\n",
+        "    #examples=examples,\n",
+        "    example_selector = example_selector_mmr,\n",
+        "    example_prompt = example_prompt,\n",
+        "    suffix=\"Question: {input}\",\n",
+        "    input_variables=[\"input\"]\n",
+        ")\n",
+        "\n",
+        "prompt_ngram = FewShotPromptTemplate(\n",
+        "    #examples=examples,\n",
+        "    example_selector = example_selector_ngram,\n",
+        "    example_prompt = example_prompt,\n",
+        "    suffix=\"Question: {input}\",\n",
+        "    input_variables=[\"input\"]\n",
+        ")"
+      ],
+      "metadata": {
+        "id": "k1laCoAPxhcH"
+      },
+      "execution_count": 18,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# First using the Example Set to test without an example selector, just the base OpenAI model.\n",
+        "from langchain.llms import OpenAI\n",
+        "base_llm = OpenAI()\n",
+        "base_llm(prompt.format(input=f\"\"\" Please grade the following students' answers to the question ({q2})\n",
+        "     on an A-, B-, C-level grading scale: student 1: {example1}, student 2: {example2}, and student 3: {example3}. Grade based on the following advice from the professor:\n",
+        "     One aspect of it was being specific. The poor answers (C) have a lot of platitudes, the better answers give\n",
+        "     specific examples (B and A). Secondly, they should discuss automation and/or prediction specifically. Those are\n",
+        "     the things that ML does (A and B answers), it is not 'technology' broadly (what a C answer might say).\n",
+        "     The difference between A and B answers should be in their complexity, specificity, thoughfulness\n",
+        "     , and general insight on the subject. However, keep in mind that these are short answer questions.\n",
+        "     The number of examples or the legnth of the response is not as important as the content itself. After assigning your grade, provide a brief\n",
+        "     explanation on your reasoning for assigning the grade based on the professor's advice, and\n",
+        "     what differentiated the answers from the higher grade level (if applicable). There should be three grades assigned in your response, and assign based on the order the\n",
+        "     answers were passed in.\"\"\"))"
+      ],
+      "metadata": {
+        "id": "rUdx2axQuec1"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# Example Selector 1:\n",
+        "\n",
+        "base_llm(prompt_mmr.format(input=f\"\"\" Please grade the following students' answers to the question ({q2})\n",
+        "     on an A-, B-, C-level grading scale: student 1: {example1}, student 2: {example2}, and student 3: {example3}. Grade based on the following advice from the professor:\n",
+        "     One aspect of it was being specific. The poor answers (C) have a lot of platitudes, the better answers give\n",
+        "     specific examples (B and A). Secondly, they should discuss automation and/or prediction specifically. Those are\n",
+        "     the things that ML does (A and B answers), it is not 'technology' broadly (what a C answer might say).\n",
+        "     The difference between A and B answers should be in their complexity, specificity, thoughfulness\n",
+        "     , and general insight on the subject. However, keep in mind that these are short answer questions.\n",
+        "     The number of examples or the legnth of the response is not as important as the content itself. After assigning your grade, provide a brief\n",
+        "     explanation on your reasoning for assigning the grade based on the professor's advice, and\n",
+        "     what differentiated the answers from the higher grade level (if applicable). There should be three grades assigned in your response, and assign based on the order the\n",
+        "     answers were passed in.\"\"\"))"
+      ],
+      "metadata": {
+        "id": "fe24IOslyITA"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# Example selector 2:\n",
+        "\n",
+        "base_llm(prompt_ngram.format(input=f\"\"\" Please grade the following students' answers to the question ({q2})\n",
+        "     on an A-, B-, C-level grading scale: student 1: {example1}, student 2: {example2}, and student 3: {example3}. Grade based on the following advice from the professor:\n",
+        "     One aspect of it was being specific. The poor answers (C) have a lot of platitudes, the better answers give\n",
+        "     specific examples (B and A). Secondly, they should discuss automation and/or prediction specifically. Those are\n",
+        "     the things that ML does (A and B answers), it is not 'technology' broadly (what a C answer might say).\n",
+        "     The difference between A and B answers should be in their complexity, specificity, thoughfulness\n",
+        "     , and general insight on the subject. However, keep in mind that these are short answer questions.\n",
+        "     The number of examples or the legnth of the response is not as important as the content itself. After assigning your grade, provide a brief\n",
+        "     explanation on your reasoning for assigning the grade based on the professor's advice, and\n",
+        "     what differentiated the answers from the higher grade level (if applicable). There should be three grades assigned in your response, and assign based on the order the\n",
+        "     answers were passed in.\"\"\"))"
+      ],
+      "metadata": {
+        "id": "dXwJa30wyL4S"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "### Results from the ngram example selector with the following prompt (yielded the best results (relatively):\n",
+        "\n",
+        "\n",
+        "\n",
+        "```\n",
+        "input=f\"\"\" Please grade the following student's answer to the question ({q2})\n",
+        "     on an A-, B-, C-level grading scale: {example3}. Grade based on the following advice from the professor:\n",
+        "     One aspect of it was being specific. The poor answers have a lot of platitudes, the better answers give\n",
+        "     specific examples. Secondly, they should discuss automation and/or prediction specifically. Those are\n",
+        "     the things that ML does, it is not 'technology' broadly. The best answers\n",
+        "     (A) should be more complex and specific in answering the question and meeting that criteria\n",
+        "     than the better answers (B), though length may not be a determinant. After assigning your grade, provide a brief\n",
+        "     explanation on your reasoning for assigning the grade based on the professor's advice.\"\"\"\n",
+        "```\n",
+        "\n",
+        "\n",
+        "\n",
+        "1. Example 1 (A-level grade): \"This student should recieve a B-level grade. The student's answer includes specific examples of how machine learning can be used in business to increase efficiency and profit margins, as well as how machine learning can automate processes previously done by humans. The student also mentions the importance of prediction in machine learning, which is one of its core functions.\" (incorrect)\n",
+        "2. Example 2 (B-level grade): \"This student should recieve a B-level grade. The student's answer provides specific examples of how ML can be used in business such as NLP and predictive analysis. The student also references automation and prediction which are key aspects of ML.\" (correct)\n",
+        "3. Example 3 (C-level grade): \"This student should recieve a B-level grade. The student's answer provides a specific example about how machine learning can help businesses save time and money by providing more accurate predictions. It also mentions the three main innovations that machine learning brings to the table (prediction is cheap, more accurate, and automated).\" (incorrect)"
+      ],
+      "metadata": {
+        "id": "bKqtzH8hcoHD"
+      }
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "# Exploring Chat Prompt Templates and System/Human Messages\n",
+        "This section is unfinalized as we are not sure if it is neccessary for the goals of this project, and we ran into errors which we were unsuccessful in debugging. This approach may yield better results but would need to be further devloped and debugged."
+      ],
+      "metadata": {
+        "id": "YGWq7cw697Wv"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from langchain.chat_models import ChatOpenAI\n",
+        "from langchain import PromptTemplate, LLMChain\n",
+        "from langchain.prompts.chat import (\n",
+        "    ChatPromptTemplate,\n",
+        "    SystemMessagePromptTemplate,\n",
+        "    AIMessagePromptTemplate,\n",
+        "    HumanMessagePromptTemplate,\n",
+        ")\n",
+        "from langchain.schema import AIMessage, HumanMessage, SystemMessage\n",
+        "\n",
+        "\n",
+        "\n",
+        "from langchain.prompts import (\n",
+        "    ChatPromptTemplate,\n",
+        "    PromptTemplate,\n",
+        "    SystemMessagePromptTemplate,\n",
+        "    AIMessagePromptTemplate,\n",
+        "    HumanMessagePromptTemplate,\n",
+        ")\n",
+        "from langchain.schema import (\n",
+        "    AIMessage,\n",
+        "    HumanMessage,\n",
+        "    SystemMessage\n",
+        ")"
+      ],
+      "metadata": {
+        "id": "NzK7Yo_OEqlG"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "template=\"You are a helpful assistant that grades student responses ({answer}) to questions ({question}) about machine learning.\"\n",
+        "system_message_prompt = SystemMessagePromptTemplate.from_template(template)\n",
+        "human_template=\"{text}\"\n",
+        "human_message_prompt = HumanMessagePromptTemplate.from_template(human_template)"
+      ],
+      "metadata": {
+        "id": "XEsYhpKMFcud"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "fprompt=PromptTemplate(\n",
+        "    template=\"You are a helpful assistant that grades student responses ({answer}) to questions ({question}) about machine learning.\",\n",
+        "    input_variables=[\"question\", \"answer\"],\n",
+        ")\n",
+        "system_message_prompt = SystemMessagePromptTemplate(prompt=prompt)"
+      ],
+      "metadata": {
+        "id": "En5SM-ZHCT8l"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "chat_prompt = ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt])\n",
+        "\n",
+        "# get a chat completion from the formatted messages\n",
+        "chat_prompt.format_prompt(input = [q2, example1], text= \"The student should recieve an A-level grade.\" ).to_messages()"
+      ],
+      "metadata": {
+        "id": "W2zIxXxDFNIh"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "example_human = SystemMessagePromptTemplate.from_template(\n",
+        "    \"Machine learning is awesome\", additional_kwargs={\"name\": \"example_user\"}\n",
+        ")\n",
+        "example_ai = SystemMessagePromptTemplate.from_template(\n",
+        "    \"This student should reieve a C-level grade.\", additional_kwargs={\"name\": \"example_assistant\"}\n",
+        ")\n",
+        "human_template = \"{text}\"\n",
+        "human_message_prompt = HumanMessagePromptTemplate.from_template(human_template)"
+      ],
+      "metadata": {
+        "id": "b_r2Y7YXTa0R"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "chat = ChatOpenAI(temperature=0)"
+      ],
+      "metadata": {
+        "id": "R2XA1z27cl6V"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "chat_prompt = ChatPromptTemplate.from_messages(\n",
+        "    [system_message_prompt, example_human, example_ai, human_message_prompt]\n",
+        ")\n",
+        "chain = LLMChain(llm=chat, prompt=chat_prompt)\n",
+        "# get a chat completion from the formatted messages\n",
+        "text_dict = {'text': 'Please grade the following students response to question 2 based on the A-, B-, C-level grading scale'}\n",
+        "chain.run(text_dict)"
+      ],
+      "metadata": {
+        "id": "3WNMa70jVuf5"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "\n",
+        "query = f\"\"\" Please grade the following students answer: {example1} to the question ({q2}).\n",
+        "The uploaded json should serve as as examples of A, B, and C level answers. In the document, the\n",
+        "original question is printed, as well as examples of previous student answers that have recieved\n",
+        "A, B, and C grades (labeled accordingly)\"\"\"\n",
+        "grades = qa.run(query)\n",
+        "print(grades)"
+      ],
+      "metadata": {
+        "id": "naKuTxKa2-U8"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## Conclusions based on Question 2\n",
+        "\n",
+        "Using different types of chains and example selectors made minor differences, but the results were still not entirely correct, and also inconsistent. Using a persona prompt as the system message was semi-useful, as was adding the grading criteria to the prompt. But it did not remedy our problem.\n",
+        "\n",
+        "Moving forward:\n",
+        "\n",
+        "1. Grading criteria helped, but for longer/more complex questions, more context may be needed (i.e., more specifics on what seperates an A from B, B from C, etc. The model did not perform as well in distinguishing between these levels/did not seem to understand the nuances of the grading criteria).\n",
+        "2. The model would likely perform better with a better example set/more context to be trained on.\n",
+        "\n"
+      ],
+      "metadata": {
+        "id": "C7kDC2pe7bNd"
+      }
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "provenance": [],
+      "include_colab_link": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "name": "python",
+      "version": "3.10.6"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

lo-achievement/instructor_intr_notebook_grading_training.ipynb

@@ -0,0 +1,737 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "view-in-github",
+        "colab_type": "text"
+      },
+      "source": [
+        "<a href=\"https://colab.research.google.com/github/vanderbilt-data-science/lo-achievement/blob/adding_grading_levels_to_instructor_nb/instructor_intr_notebook_grading_training.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "collapsed": true,
+        "id": "brzvVeAsYiG2"
+      },
+      "source": [
+        "<a href=\"https://colab.research.google.com/github/vanderbilt-data-science/lo-achievement/blob/main/instructor_intr_notebook.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WMKrKfx8_3fc"
+      },
+      "source": [
+        "# Instructor Grading and Assessment\n",
+        "This notebook executes grading of student submissions based on the examples provided in the [Wiki](https://github.com/vanderbilt-data-science/lo-achievement/wiki/Examples-of-great,-good,-and-poor-answers-to-questions) from Dr. Jesse Blocher. In this iteration, we use the Unstructured File Loader, which cannot proccess .json files (the preferred format). We are working on finding a file loader that allows .json. In this version of the notebook, the model has only been trained on Question 2 from the notebook.\n",
+        "\n",
+        "To train the model, we used 2 out of the three student example from each grade brack and inputted into a .pdf with clearly defined levels. Then, we used the excluded answers to test the accuracy of the model's grading."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## Grading based on A, B, and C-level answers from previous students to Question 2 from the [Wiki](https://github.com/vanderbilt-data-science/lo-achievement/wiki/Examples-of-great,-good,-and-poor-answers-to-questions):\n",
+        "\n",
+        "**Question 2:** Why is machine learning so important for businesses? Answer this question generally (i.e. such that it applies to many or at least most businesses)."
+      ],
+      "metadata": {
+        "id": "ZTkNQ-dL5iO5"
+      }
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "### Creating .json file from case examples (Question 2)\n",
+        "The purpose of this cell is to create a json file based on the previously submitted, graded work of students based on the case file provided by Dr. Blocher in the Wiki"
+      ],
+      "metadata": {
+        "id": "TYlGEusr64kA"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# Context: question\n",
+        "\n",
+        "q2 = 'Question 2: Why is machine learning so important for businesses? Answer this question generally (i.e. such that it applies to many or at least most businesses).'"
+      ],
+      "metadata": {
+        "id": "-kmMFUaLs_Q1"
+      },
+      "execution_count": 1,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# A-level answers\n",
+        "\n",
+        "A_answer_1 = 'Machine learning is extremely important tool for businesses. It can be used in a variety of ways, but most importantly, it can be used to identify patterns within their data that might not otherwise be identified by human beings. For example, it can understand customer behaviors, optimize logistics, and expand efficiencies throughout the business. Machine learning does not get tired, meaning it can work as long as you want it to. It can sift through massive amounts of data, that no human being can look through in an efficient manner. Machine learning can be used as a tool to identify anomalies when something needs to be checked to save or gain money. The predictions that companies gain from machine learning are cheap, accurate, and automate. These machine learning algorithms can be brought to larger scales to encompass the whole business and its operations. It is important to note, Machine learning is just predictions. Predictions to understand important patterns that could make or break a company since they understand the patterns of their business more. It is an amazing tool, but should be used wisely and carefully because if not, it can expensive, useless, and straight up wrong.'\n",
+        "A_answer_2 = 'Machine learning is important for most of the sectors in business. Overall, it gives the company of an overview about what would be the trend for their business industry, and analyze the customer behavior to help business segment their customers groups. Today, many companies have a vast amount of information generated by behavior, computer, events, people, and devices. This massive amount of data is difficult for human to handle, and even if human manages it, it is not profitable as human labor is expensive. Thanks to machine learning, companies can utilize their in-house or even third-party data to make something useful for their business. In medical analysis, for example, with human, it takes a very long time to find patterns in thousands of MRI scans. On the other hand, machines can detect patterns in seconds by entering data as long as the information is correctly labeled or trained properly. Another example would be segmenting customer group. In marketing department, the business could use unsupervised machine learning to cluster their customer segments to generate personalized contents that are relevant for each of individuals.'\n",
+        "\n",
+        "# List creation\n",
+        "\n",
+        "A_answers_list = [A_answer_1, A_answer_2]\n"
+      ],
+      "metadata": {
+        "id": "yQT6aExSr1dP"
+      },
+      "execution_count": 2,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# B-level answers\n",
+        "\n",
+        "B_answer_1 = 'Companies use ML models to improve different aspects of their business, like manufacturing, hiring, deployment, advertising, etc. The main goal is to improve productive and increase profitability of the company. The ML models are fed with company and externally available data to help the company optimize its departments and in turn become more financially successful/ productive. For example, using purchasing history, the company can predict who to advertise products to, to increase sales.'\n",
+        "B_answer_2 = 'Machine learning allows business to have automated decision, scale, predictive analysis and performance. Machine learning also helps a business have a data strategy. This is how a firm uses data, data infrastructure, governance, etc. to accomplish its strategic goals and maintain/grow their competitive advantage within their industry.'\n",
+        "B_answer_3 = 'The short answer is ML can help make decisions for businesses. To be clarified, ML does not make decisions for businesses. I mean it can, but people have not trusted ML enough yet and ML has not been that good to let it directly make business decisions. Business people only use ML to help themselves get intuitions of how decisions should be made and make predictions of results they might get based on their decisions. For example, if a business tries to launch a new product, it will use ML to test whether it will work or not on a small scale before it is introduced to a large scale. People called this step piloting. In this step, people collect data that is generated by using the pilot product and analyze their next move. They could tweak some features based on the feedback. If they think the data in their interest shows the product performs well, they might predict that this product will be successful when it is introduced on a large scale. Then, they will launch it.'\n",
+        "# List creation\n",
+        "\n",
+        "B_answers_list = [B_answer_1, B_answer_2, B_answer_3]"
+      ],
+      "metadata": {
+        "id": "KB1CmeRwtRvf"
+      },
+      "execution_count": 3,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# C-level answers\n",
+        "\n",
+        "C_answer_1 = 'Machine learning powers many of the services we use today, such as the recommendation systems of Spotify and Netflix; search engines such as Google and Bing; social media such as TikTok and Instagram; voices such as Siri and Alexa, the list can go on. All these examples show that machine learning is already starting to play a pivotal role in today\"s data-rich world. Machines can help us sift through useful information that can lead to big breakthroughs, and we have seen the widespread use of this technology in various industries such as finance, healthcare, insurance, manufacturing, transformational change, etc.'\n",
+        "C_answer_3 = 'As technology advanced, there are tons of new data generated and stored. All industries experienced this surge in data, including business. There is a huge amount of business data stored and waited in the database of each firm and they need solutions to utilize these data. Machine learning is a very promising approach for firms to puts these data in and output a meaning pattern or result that could help the firms with their existing work. This could turn into a working product or provide insights that could enhance the efficiency of the company’s workflow. With machine learning, a firm could either enter a new market with the new product or save time and effort with the meaningful insights. Achieving these possibilities with the data they already owned is a low effort but high reward action. This is the reason machine learning is valued by many businesses recently.'\n",
+        "\n",
+        "# List creation\n",
+        "\n",
+        "C_answers_list = [C_answer_1, C_answer_3]"
+      ],
+      "metadata": {
+        "id": "3diAz43othjc"
+      },
+      "execution_count": 4,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "Q_and_A = [\"Question:\", q2, \"A-level Answers\", A_answers_list, \"B-level Answers\", B_answers_list, \"C-level Answers\", C_answers_list]"
+      ],
+      "metadata": {
+        "id": "oAnrMSU6u9do"
+      },
+      "execution_count": 5,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "import json\n",
+        "from google.colab import files\n",
+        "\n",
+        "def save_example_answers(examples, filename='wiki_ABC_Q2examples.json'):\n",
+        "  with open(filename, 'w') as file:\n",
+        "    json.dump(examples, file)\n",
+        "  files.download(filename)\n",
+        "\n",
+        "save_example_answers(Q_and_A)"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 17
+        },
+        "id": "B16iYMEnri9s",
+        "outputId": "3e565b3d-804c-4b5e-acc8-efeb955c6c14"
+      },
+      "execution_count": 6,
+      "outputs": [
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "<IPython.core.display.Javascript object>"
+            ],
+            "application/javascript": [
+              "\n",
+              "    async function download(id, filename, size) {\n",
+              "      if (!google.colab.kernel.accessAllowed) {\n",
+              "        return;\n",
+              "      }\n",
+              "      const div = document.createElement('div');\n",
+              "      const label = document.createElement('label');\n",
+              "      label.textContent = `Downloading \"${filename}\": `;\n",
+              "      div.appendChild(label);\n",
+              "      const progress = document.createElement('progress');\n",
+              "      progress.max = size;\n",
+              "      div.appendChild(progress);\n",
+              "      document.body.appendChild(div);\n",
+              "\n",
+              "      const buffers = [];\n",
+              "      let downloaded = 0;\n",
+              "\n",
+              "      const channel = await google.colab.kernel.comms.open(id);\n",
+              "      // Send a message to notify the kernel that we're ready.\n",
+              "      channel.send({})\n",
+              "\n",
+              "      for await (const message of channel.messages) {\n",
+              "        // Send a message to notify the kernel that we're ready.\n",
+              "        channel.send({})\n",
+              "        if (message.buffers) {\n",
+              "          for (const buffer of message.buffers) {\n",
+              "            buffers.push(buffer);\n",
+              "            downloaded += buffer.byteLength;\n",
+              "            progress.value = downloaded;\n",
+              "          }\n",
+              "        }\n",
+              "      }\n",
+              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
+              "      const a = document.createElement('a');\n",
+              "      a.href = window.URL.createObjectURL(blob);\n",
+              "      a.download = filename;\n",
+              "      div.appendChild(a);\n",
+              "      a.click();\n",
+              "      div.remove();\n",
+              "    }\n",
+              "  "
+            ]
+          },
+          "metadata": {}
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "<IPython.core.display.Javascript object>"
+            ],
+            "application/javascript": [
+              "download(\"download_8ec6f24c-9653-4335-8c1b-766926434399\", \"wiki_ABC_Q2examples.json\", 5931)"
+            ]
+          },
+          "metadata": {}
+        }
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## **Start here** to interact with model"
+      ],
+      "metadata": {
+        "id": "PpKRr5Vw4_0F"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "! pip install -q langchain=='0.0.229' openai gradio numpy chromadb tiktoken unstructured pdf2image pydantic==\"1.10.8\" jq"
+      ],
+      "metadata": {
+        "id": "UJi1Oy0CyPHD"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# import necessary libraries here\n",
+        "from getpass import getpass\n",
+        "from langchain.llms import OpenAI as openai\n",
+        "from langchain.chat_models import ChatOpenAI\n",
+        "from langchain.prompts import PromptTemplate\n",
+        "from langchain.document_loaders import TextLoader\n",
+        "from langchain.indexes import VectorstoreIndexCreator\n",
+        "from langchain.text_splitter import CharacterTextSplitter\n",
+        "from langchain.embeddings import OpenAIEmbeddings\n",
+        "from langchain.schema import SystemMessage, HumanMessage, AIMessage\n",
+        "import numpy as np\n",
+        "import os\n",
+        "from langchain.vectorstores import Chroma\n",
+        "from langchain.document_loaders.unstructured import UnstructuredFileLoader\n",
+        "from langchain.document_loaders import UnstructuredFileLoader\n",
+        "from langchain.chains import VectorDBQA\n",
+        "from langchain.document_loaders import JSONLoader\n",
+        "import json\n",
+        "from pathlib import Path\n",
+        "from pprint import pprint"
+      ],
+      "metadata": {
+        "id": "YHytCUoExrYe"
+      },
+      "execution_count": 18,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from google.colab import files\n",
+        "uploaded = files.upload()"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 74
+        },
+        "id": "Wpt6qsmEw8WP",
+        "outputId": "4563fa62-5245-4115-ecb9-326353dba29c"
+      },
+      "execution_count": 7,
+      "outputs": [
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "<IPython.core.display.HTML object>"
+            ],
+            "text/html": [
+              "\n",
+              "     <input type=\"file\" id=\"files-6f630631-16ee-49df-8de9-29bfa3ec5f7b\" name=\"files[]\" multiple disabled\n",
+              "        style=\"border:none\" />\n",
+              "     <output id=\"result-6f630631-16ee-49df-8de9-29bfa3ec5f7b\">\n",
+              "      Upload widget is only available when the cell has been executed in the\n",
+              "      current browser session. Please rerun this cell to enable.\n",
+              "      </output>\n",
+              "      <script>// Copyright 2017 Google LLC\n",
+              "//\n",
+              "// Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+              "// you may not use this file except in compliance with the License.\n",
+              "// You may obtain a copy of the License at\n",
+              "//\n",
+              "//      http://www.apache.org/licenses/LICENSE-2.0\n",
+              "//\n",
+              "// Unless required by applicable law or agreed to in writing, software\n",
+              "// distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+              "// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+              "// See the License for the specific language governing permissions and\n",
+              "// limitations under the License.\n",
+              "\n",
+              "/**\n",
+              " * @fileoverview Helpers for google.colab Python module.\n",
+              " */\n",
+              "(function(scope) {\n",
+              "function span(text, styleAttributes = {}) {\n",
+              "  const element = document.createElement('span');\n",
+              "  element.textContent = text;\n",
+              "  for (const key of Object.keys(styleAttributes)) {\n",
+              "    element.style[key] = styleAttributes[key];\n",
+              "  }\n",
+              "  return element;\n",
+              "}\n",
+              "\n",
+              "// Max number of bytes which will be uploaded at a time.\n",
+              "const MAX_PAYLOAD_SIZE = 100 * 1024;\n",
+              "\n",
+              "function _uploadFiles(inputId, outputId) {\n",
+              "  const steps = uploadFilesStep(inputId, outputId);\n",
+              "  const outputElement = document.getElementById(outputId);\n",
+              "  // Cache steps on the outputElement to make it available for the next call\n",
+              "  // to uploadFilesContinue from Python.\n",
+              "  outputElement.steps = steps;\n",
+              "\n",
+              "  return _uploadFilesContinue(outputId);\n",
+              "}\n",
+              "\n",
+              "// This is roughly an async generator (not supported in the browser yet),\n",
+              "// where there are multiple asynchronous steps and the Python side is going\n",
+              "// to poll for completion of each step.\n",
+              "// This uses a Promise to block the python side on completion of each step,\n",
+              "// then passes the result of the previous step as the input to the next step.\n",
+              "function _uploadFilesContinue(outputId) {\n",
+              "  const outputElement = document.getElementById(outputId);\n",
+              "  const steps = outputElement.steps;\n",
+              "\n",
+              "  const next = steps.next(outputElement.lastPromiseValue);\n",
+              "  return Promise.resolve(next.value.promise).then((value) => {\n",
+              "    // Cache the last promise value to make it available to the next\n",
+              "    // step of the generator.\n",
+              "    outputElement.lastPromiseValue = value;\n",
+              "    return next.value.response;\n",
+              "  });\n",
+              "}\n",
+              "\n",
+              "/**\n",
+              " * Generator function which is called between each async step of the upload\n",
+              " * process.\n",
+              " * @param {string} inputId Element ID of the input file picker element.\n",
+              " * @param {string} outputId Element ID of the output display.\n",
+              " * @return {!Iterable<!Object>} Iterable of next steps.\n",
+              " */\n",
+              "function* uploadFilesStep(inputId, outputId) {\n",
+              "  const inputElement = document.getElementById(inputId);\n",
+              "  inputElement.disabled = false;\n",
+              "\n",
+              "  const outputElement = document.getElementById(outputId);\n",
+              "  outputElement.innerHTML = '';\n",
+              "\n",
+              "  const pickedPromise = new Promise((resolve) => {\n",
+              "    inputElement.addEventListener('change', (e) => {\n",
+              "      resolve(e.target.files);\n",
+              "    });\n",
+              "  });\n",
+              "\n",
+              "  const cancel = document.createElement('button');\n",
+              "  inputElement.parentElement.appendChild(cancel);\n",
+              "  cancel.textContent = 'Cancel upload';\n",
+              "  const cancelPromise = new Promise((resolve) => {\n",
+              "    cancel.onclick = () => {\n",
+              "      resolve(null);\n",
+              "    };\n",
+              "  });\n",
+              "\n",
+              "  // Wait for the user to pick the files.\n",
+              "  const files = yield {\n",
+              "    promise: Promise.race([pickedPromise, cancelPromise]),\n",
+              "    response: {\n",
+              "      action: 'starting',\n",
+              "    }\n",
+              "  };\n",
+              "\n",
+              "  cancel.remove();\n",
+              "\n",
+              "  // Disable the input element since further picks are not allowed.\n",
+              "  inputElement.disabled = true;\n",
+              "\n",
+              "  if (!files) {\n",
+              "    return {\n",
+              "      response: {\n",
+              "        action: 'complete',\n",
+              "      }\n",
+              "    };\n",
+              "  }\n",
+              "\n",
+              "  for (const file of files) {\n",
+              "    const li = document.createElement('li');\n",
+              "    li.append(span(file.name, {fontWeight: 'bold'}));\n",
+              "    li.append(span(\n",
+              "        `(${file.type || 'n/a'}) - ${file.size} bytes, ` +\n",
+              "        `last modified: ${\n",
+              "            file.lastModifiedDate ? file.lastModifiedDate.toLocaleDateString() :\n",
+              "                                    'n/a'} - `));\n",
+              "    const percent = span('0% done');\n",
+              "    li.appendChild(percent);\n",
+              "\n",
+              "    outputElement.appendChild(li);\n",
+              "\n",
+              "    const fileDataPromise = new Promise((resolve) => {\n",
+              "      const reader = new FileReader();\n",
+              "      reader.onload = (e) => {\n",
+              "        resolve(e.target.result);\n",
+              "      };\n",
+              "      reader.readAsArrayBuffer(file);\n",
+              "    });\n",
+              "    // Wait for the data to be ready.\n",
+              "    let fileData = yield {\n",
+              "      promise: fileDataPromise,\n",
+              "      response: {\n",
+              "        action: 'continue',\n",
+              "      }\n",
+              "    };\n",
+              "\n",
+              "    // Use a chunked sending to avoid message size limits. See b/62115660.\n",
+              "    let position = 0;\n",
+              "    do {\n",
+              "      const length = Math.min(fileData.byteLength - position, MAX_PAYLOAD_SIZE);\n",
+              "      const chunk = new Uint8Array(fileData, position, length);\n",
+              "      position += length;\n",
+              "\n",
+              "      const base64 = btoa(String.fromCharCode.apply(null, chunk));\n",
+              "      yield {\n",
+              "        response: {\n",
+              "          action: 'append',\n",
+              "          file: file.name,\n",
+              "          data: base64,\n",
+              "        },\n",
+              "      };\n",
+              "\n",
+              "      let percentDone = fileData.byteLength === 0 ?\n",
+              "          100 :\n",
+              "          Math.round((position / fileData.byteLength) * 100);\n",
+              "      percent.textContent = `${percentDone}% done`;\n",
+              "\n",
+              "    } while (position < fileData.byteLength);\n",
+              "  }\n",
+              "\n",
+              "  // All done.\n",
+              "  yield {\n",
+              "    response: {\n",
+              "      action: 'complete',\n",
+              "    }\n",
+              "  };\n",
+              "}\n",
+              "\n",
+              "scope.google = scope.google || {};\n",
+              "scope.google.colab = scope.google.colab || {};\n",
+              "scope.google.colab._files = {\n",
+              "  _uploadFiles,\n",
+              "  _uploadFilesContinue,\n",
+              "};\n",
+              "})(self);\n",
+              "</script> "
+            ]
+          },
+          "metadata": {}
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Saving wiki_ABC_Q2examples (2).json to wiki_ABC_Q2examples (2).json\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# setup open AI api key\n",
+        "openai_api_key = getpass()"
+      ],
+      "metadata": {
+        "id": "jVPEFX3ixJnM"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "os.environ[\"OPENAI_API_KEY\"] = openai_api_key\n",
+        "openai.api_key = openai_api_key"
+      ],
+      "metadata": {
+        "id": "MO7VuGmrxVAr"
+      },
+      "execution_count": 19,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "mdl_name = 'gpt-3.5-turbo-0301'"
+      ],
+      "metadata": {
+        "id": "4Thxj6Gk1zVS"
+      },
+      "execution_count": 20,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "llm = ChatOpenAI(model='gpt-3.5-turbo-16k')\n",
+        "messages = [\n",
+        "    SystemMessage(content=\"You are a helpful assistant.\"),\n",
+        "    HumanMessage(content=\"\")\n",
+        "]"
+      ],
+      "metadata": {
+        "id": "Sgq9aVqpxZnK"
+      },
+      "execution_count": 36,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "file_path='/content/wiki_ABC_Q2examples (2).json'"
+      ],
+      "metadata": {
+        "id": "Ak7X_ZRba48F"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "data = json.loads(Path(file_path).read_text())\n",
+        "data = str(data)"
+      ],
+      "metadata": {
+        "id": "PKqQROVMc6BP"
+      },
+      "execution_count": 26,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# Contruct Vector Store\n",
+        "text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)\n",
+        "texts = text_splitter.split_text(data)\n",
+        "\n",
+        "embeddings = OpenAIEmbeddings()\n",
+        "\n",
+        "db = Chroma.from_texts(texts, embeddings)\n",
+        "\n",
+        "qa = VectorDBQA.from_chain_type(llm=ChatOpenAI(model_name = mdl_name), chain_type=\"stuff\", vectorstore=db, k=1)"
+      ],
+      "metadata": {
+        "id": "i2NadtLlcxur"
+      },
+      "execution_count": 37,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# A level\n",
+        "example1 = 'The most powerful aspect of machine learning is its ability to automate processes. If the goal is well defined and repeatable, an algorithm can be trained to perform that task far faster than any human and often with more reliability. Because of this, businesses can implement machine learning algorithms to solve problems human labor was previously required for, whether that be mental or physical labor. Although implementing machine learning often has high initial costs, because computers do not require payment outside of maintaining their operation, in the long run companies can save money by either making their workers’ tasks more efficient or by entirely automating tasks. This increases the profit margins for those firms.'\n",
+        "# B level\n",
+        "example2 ='ML systems can help improve access to data while managing compliance. It helps businesspeople deal with data in a more efficient way, since there is numerous data generated per second in such industry. If they use some of the simple classification models with high accuracy, it will save them a lot of time on data cleaning and validation, which are kind of repetitive and time-consuming. For example, NLP provides an easy way for personnel to query business information, understand business processes, and discover new relationships between business data, ideas based on intuition and insight often emerge. Using models to do prediction helps people making wiser decision. Since models can handle way more data than human, newly collected data can feed in the model and get some predictive result as a reference to the decision makers. This is significant because in this industry, time is precious, and traders must decide quickly and precisely. A little negligence will lead to a big mistake, lose a lot of money, and even affect the company\"s reputation. Models can see patterns that are not easy for human to spot, which is also valuable for modify the way people doing analysis and interpret.'\n",
+        "# C level\n",
+        "example3 = 'The machine learning model (or one a broader view, artificial intelligence) is about prediction. According to the lecture, there are tree main innovations in it. Prediction is cheap, more accurate and automated. As a result, armed with machine learning, businesses could automatically gain much more accurate and powerful capabilities in forecasting, leading to a big savings both in time and money.'\n",
+        "\n",
+        "# Randomized list of answers\n",
+        "training_answers = [example2, example1, example3]"
+      ],
+      "metadata": {
+        "id": "CJxUs8lG12kd"
+      },
+      "execution_count": 38,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "query = f\"\"\" Please grade the following student answers: {training_answers} to the question ({q2}).\n",
+        "The uploaded pdf should serve as as examples of A, B, and C level answers. In the document, the\n",
+        "original question is printed, as well as examples of previous student answers that have recieved\n",
+        "A, B, and C grades (labeled accordingly)\"\"\"\n",
+        "\n",
+        "query_prefix = \"\"\" The uploaded pdf should serve as as examples of A, B, and C level answers.\n",
+        "In the document, the original question is printed, as well as examples of previous student answers that have recieved\n",
+        "A, B, and C grades (labeled accordingly).\"\"\"\n",
+        "answer = qa.run(query_prefix + query)\n",
+        "print(answer)"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "naKuTxKa2-U8",
+        "outputId": "fc10a157-071d-4a09-bf4d-315fec1d53e9"
+      },
+      "execution_count": 56,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "The first answer would be a B-level answer. It mentions the efficiency of using machine learning for data cleaning and validation, as well as the ability to handle large amounts of data and make predictions. However, it could be more specific in terms of how machine learning can benefit businesses in various industries.\n",
+            "\n",
+            "The second answer would be an A-level answer. It highlights the ability of machine learning to automate processes and save money in the long run, as well as mentioning the reliability and efficiency of algorithms compared to human labor.\n",
+            "\n",
+            "The third answer would also be an A-level answer. It accurately describes the main innovation of machine learning as being prediction, and how it can lead to cost savings in time and money for businesses. It also hints at the potential for machine learning to improve forecasting capabilities.\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## Conclusions based on Question 2\n",
+        "\n",
+        "Moving forward:\n",
+        "\n",
+        "\n",
+        "1.   Train the model on the other questions provided in the case example\n",
+        "2.   Request more student answers from Dr. Blocher\n",
+        "3. Request a grading rubric from Dr. Blocher for the case examples, which may help the model gain more context\n",
+        "\n"
+      ],
+      "metadata": {
+        "id": "C7kDC2pe7bNd"
+      }
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "### Ouput for query with .pdf\n",
+        "\n",
+        "We also experimenting with using a .pdf file (before we figured out how to parse .json files). Here is one set of results.\n",
+        "\n",
+        "The first answer can be graded as a B. It touches on the benefits of machine learning such as improving access to data, efficiency in data management, and faster decision-making. However, the answer could be strengthened by providing more specific examples of how machine learning has benefited businesses.\n",
+        "\n",
+        "The second answer can be graded as an A. It provides a clear and concise explanation of how machine learning can automate processes and save companies money in the long run. The answer also acknowledges the initial costs of implementing machine learning but emphasizes the potential for increased profit margins.\n",
+        "\n",
+        "The third answer can also be graded as an A. It highlights the main innovation of machine learning which is prediction and how it can lead to cost and time savings for businesses. The answer is well-organized and provides a clear explanation of why machine learning is important for businesses in general.\n",
+        "\n",
+        "**Results:**\n",
+        "\n",
+        "The model did not perform as well as expected. It did not successfully grade any of the questions. The first qyestion should have been graded as an A, second as a B, and third as a C."
+      ],
+      "metadata": {
+        "id": "iJTiGX4aZ91_"
+      }
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "### Output for .json queries\n",
+        "\n",
+        "**First query:**\n",
+        "\n",
+        "The first answer would be a B-level answer. It touches on the efficiency and time-saving benefits of machine learning, but could benefit from more specific examples and a deeper explanation of how it can improve business outcomes.\n",
+        "\n",
+        "The second answer would be an A-level answer. It provides a clear explanation of how machine learning can automate processes and improve efficiency, leading to cost savings and increased profits for businesses.\n",
+        "\n",
+        "The third answer would also be an A-level answer. It focuses on the predictive power of machine learning and how it can save time and money for businesses by providing more accurate forecasts. It also mentions the automation benefits of machine learning.\n",
+        "\n",
+        "Notes: Same results as first query using .pdf.\n",
+        "\n",
+        "**Second query:**\n",
+        "\n",
+        "Answer 1: B-level answer. The answer talks about how machine learning can automate processes and solve problems that were previously done by human labor. It also mentions how implementing machine learning can save money in the long run.\n",
+        "\n",
+        "Answer 2: A-level answer. The answer discusses how machine learning can help with data cleaning and validation, which can save time and enhance efficiency. It also mentions how models can handle more data than humans and can provide predictive results for decision-makers. Additionally, it talks about how machines can see patterns that are difficult for humans to spot, which can help with analysis and interpretation.\n",
+        "\n",
+        "Answer 3: C-level answer. The answer discusses how machine learning can provide accurate and powerful capabilities in forecasting, resulting in savings in time and money. It also talks about the three main innovations in machine learning, which are prediction, accuracy, and automation. The answer provides examples of different industries that have implemented machine learning, showcasing its importance in today's data-rich world.\n",
+        "\n",
+        "Notes: Graded one answer correctky (answer C).\n",
+        "\n",
+        "**Third query**\n",
+        "\n",
+        "The first answer would be a B-level answer. While it touches on some important points such as saving time on data cleaning and validation and using models for prediction, it lacks depth and doesn't provide specific examples or applications for businesses.\n",
+        "\n",
+        "The second answer would be an A-level answer. It provides a clear and concise explanation of how machine learning can automate processes and save companies money in the long run. It also acknowledges the initial costs of implementing machine learning and how it can increase profit margins.\n",
+        "\n",
+        "The third answer would also be a B-level answer. While it mentions the three main innovations of machine learning (cheap, more accurate, and automated predictions), it doesn't provide specific examples or applications for businesses and lacks depth in its explanation.\n",
+        "\n",
+        "Notes: None of the answers correct\n",
+        "\n",
+        "### General comments:\n",
+        "\n",
+        "None of the query results were consistent. There were also intsances where the output would be something like \"As an AI language model, I do not have the capability to...\". So more training and prompt engineering is certainly needed."
+      ],
+      "metadata": {
+        "id": "FR4YMpPseoTE"
+      }
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "provenance": [],
+      "include_colab_link": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "name": "python",
+      "version": "3.10.6"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

lo-achievement/instructor_vector_store_creator.ipynb

@@ -0,0 +1,333 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "<a href=\"https://colab.research.google.com/github/vanderbilt-data-science/lo-achievement/blob/main/instructor_vector_store_creator.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Creating a Shared Vector Store (for Instructors)\n",
+    "\n",
+    "This notebook is for instructors to create a *vector store* which contains all of the information necessary for students to generate their own self-study materials using large language models. It is expected that instructors who will use this notebook know how to run and interact with a Jupyter Notebook, specifically on Google Colab.\n",
+    "\n",
+    ":::{.callout-info}\n",
+    "On Colab, there may be a pop-up saying 'Warning: This notebook was not authored by Google'. In that case, click 'Run anyways'. If you started this notebook from the Vanderbilt Data Science github, then you can trust the code in this notebook.\n",
+    ":::"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Setting Up API Access\n",
+    "Much of the following code rely on certain *APIs* (application programming interfaces) which have limited access. You will need to get an *API key* for each of those services which will be inserted into the code to let the service know you are an authorized user."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### OpenAI"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "First, you will need an **OpenAI API key**. To do this:\n",
+    "1. Visit [platform.openai.com/account/api-keys](https://platform.openai.com/account/api-keys) and sign up for an account.\n",
+    "2. Click 'Create a secret API key', and give it any name you want.\n",
+    "3. Copy the newly created key, either by right-clicking and pressing 'Copy' or using the keyboard shortcut -- Ctrl+C on Windows, Cmd+C on a Mac.\n",
+    "\n",
+    "Run the following code cell. You'll see a blank text box pop up -- paste your API key there (using the shortcut Ctrl+V on Windows, or Cmd+V if you are using a Mac) and press Enter."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "OPENAI_API_KEY = getpass(\"OpenAI API key: \")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### DeepLake\n",
+    "\n",
+    "Next, you will need to input a **DeepLake API key**, found in the DeepLake dashboard at [app.activeloop.ai](https://app.activeloop.ai).\n",
+    "\n",
+    "1. Click the link above and create an account.\n",
+    "2. After making an account, you will be prompted to set a username. Once you have set your username, copy it, run the code below, paste the username into the text box, and press Enter. (This username will be shared with students.)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "DEEPLAKE_USERNAME = input(\"DeepLake username: \")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "3. You should then be on the DeepLake dashboard. At the top, click 'Create API token'. You should see an empty table with the columns 'Name', 'Expiration date', and 'Token'.\n",
+    "4. Click the 'Create API token' button ![image.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAJYAAAAwCAYAAADzRIMRAAALoElEQVR4Xu1caXBT1xX+nvZdssFm8+DYgGPWAVMggFkyMWuJyYBDgRICDZ1JO6HJkNKkAYakEHcgSxsyQybDUkqLW9bWCVsh7GFJ2NywhuASJyQ4GCxZaF973nuWbFmSbWHJjjXv/tLy3r3nnvPdc7577nmP8VOD0AQNxFkDjACsOGtU6I7TgAAsAQgJ0YAArISoVehUAJaAgYRoQABWQtQqdBozsLhNpM8J193P4a78FO6q8/CZr8NruQp4BIW2Ww1IALGmD0S6XEjTBkPaOR+y9KGASA6GYWKeVvOB5ffCY70D+5cb4aooBSQqyDuPhTj9J5AaciFSZ0AkVccsgHDDj0MDPrcVPuttuE3kJO6eg7PyKDkKG2SZU6F89BeQqLvQVk/cbGGbBpbfB5/bDNul92AjUCmynoay12xIO+Y1exDhwvapAfe9C7B/VQLHre1QEbhU/V8k56EjgImanFCjwPJ7HXDeOQZr2VuQ6nO4jiWGR5vsVLgguTTgMX3JORZ3zQ2oBy6GvMsYcl6KRicZFVh+coO2LzfBdnE5NMPfh7LHzOTSljCbmDVgL/8nLKcXQjXoDfJg88AQHYrWIgKLA9XVD+Ao3wbdqLUU9gbHLIRwQ3JqwH3vPMwnfg1FjxlQ9flVVHCFAcvvdcJ2fQMcN/4K/eObhdCXnPho0azY0FhzZC4UOc9ClfschUV5WH+hwCKi7vjuEB4cnQ3DxP2Cp2qR+pP7ZtZzmfZPhHZsCRTdnggj9CHA8rlMMB4ogqr3AoFTJTcu4jI7lnPZrq1HyvgdEMkMIX3WAYvyVJYLK+Gz/cDxKqEJGmiOBli+JVJ1giZvaUieKwgsj+U2qj8aidTJBwRe1RyNCtdwGmD5VvXe8UgtPAmJJiOoFQ5Y7DGN5cIK+F1m6Ia/LahM0EBMGjCf/i0YmY681rLg8Q8PLEqEVpeOhG70OiGjHpNKhYtZDbAZevPxXyJ16slg4pQDlvPOcVjOLkGHwhNx1ZTH/hmc5VfA1j4zkmzKfdDZojSuQySoMyNsN3bB59JCnjUDkY5AHd9sgMdc/3A2HfLMUZBo9WB/9aMctsvHwGjHQpWZHRc52TG9jkFQ5uSh6UOVuAzZ7E7ufzQKmiFvUlZ+NHcPByzLxWKCnR2aoSua3VFjF/rxA8yHhsF5qyL0MlEuVPnHoclJi8s4DTuxX3+JzrW6QzNpEVqCX4/pDzDuWM4tCGnPM0gZOyxMXtNeBq7vw6ch6VYC/aRZBK7NqF7/LJguJejw01lhFz6MrOyYbvNSpMxcASpG+FE1y+fLSFlKaAa9Vgcs44HplOyaC+UjU+MgrAvmoyo4bnoh6bYe2sfnQqqQ0jnTFpj3zIHXVQTDrO2QhefUWjy25VQmnRgMgWHBDsha0Jv5mJhOHZ6AuNNBeKuegeGZzZA1ONjngTUZ2qfXckb2oxLWY4/BfRdQj/VD2bNxYD2MrD9mYNm/LqWk+mZKPeysA9b9HX2hL9gVl92gx1wM47YlYDp+gJSnnkd9e/hst+CXZnHh0HO/GKZ9f4as71p4btGpeef9MIzoD49lL8yHX4Dn3i0qEMqBrMdaaPPJyJy4NlgvvQD75b9QiQd9lQ2CIm8rtP16wXpuJP1+Cn62JkxBYSnnGPRDc6ky4xIeHJ4N153LZHw1JJ3XQjduLiRRKkB8vl0wbp4OpJVA03sPag5vgXKEG9o+oT6CB9b0EBC7KxfBtPtPkOZ+AV3+xageyxJFVq/tIMxHXoan6hIoV00lScW0MH8PWW01UiiwXLCcyYf9phOqEaegzmZIN/PpbHcbpwORZjY0BZugSOV9d80hhkL3ciqDOQ/Hpd2g7BKk3UuhKygMsdHDrkcuG//JNHQoulIHrLubGHT8uSUu9VTO60+i5tPdUD5GxugX3WG7q16BqXQ1/KJU8gzjIc9eDWXWSRi3zoJf+xLUg2dRAeGHsJ/dyBmKA92Dd2DauQ6S3r+j+iALXF+9SADNgGbit2AsL8H6xXt0TwYkGVSklrka6t73CeSD4PX8DKqhL9OO5d+wnSom0G9HyqSiiDzFeXMmao5uhXq0G6qcz1C9JR9+zToipgs4hQW4TTiwbERg+9CqreCAqO5TEhVYbBhsKKsqpxLGnUOpYDKXFttiiJiTcF7bSIuEQl8RhT5aCHXAWgbHuVGwlZ0lPd+Gpl9XWM/0Im/thbzvGsi7+2E/Xwj3/ZnQz/gH5Er+XjZ0izstgSwrHW5Od52gHlcJdebDwqnuPrae694WDdLn8U8Tchzr7noG6Qvi83ih9VweldlcJOX6aZVHFzgALEU9AHL3ftEJ+ln7OGWwzXpxOKz/7Y2UeQQw+u5zV8Frs5GFDfA7ilFD4FTUjtUwvLj+Nwemw8c44KlqUyzOiudgPniNQtgpKPQN5TPC9FEqXNXzg+NxYfGrPOimn+UuVqTw90TjWIz+daRMWw6ROLZQ6LwxDTXH/8UBWp3DL8gAyAMeMwAs5cAy2GjxynLPQ5+fR574Yxg3FUL0SB0f9PkOwFgyAZKcb8lzZ9QDJc/PAt41oLuWQwuoj6O4AyugIPkAfkLRWhBY9QAYzVhAb2ievAqp6A2YPn6dvJyadplu6lpK4LIGQdwQWPz3byKKEAn4AdIu7r4N2mFUlkvNW/MuHhxYAwmReLYFiDwvK+8duRUqzSYvSUWQWQMJVLQAmiDvkWUdCMP80iCfC/QhYT12fn8ezMThqHMw8oUwzFwDKY0V9P4RZiqu3Tw05GeR9N9ScIUDK46hMMBRvMw06It2Ql6vWtlZ8S48XgpL2d2IR/ChsP6KefBpV9ivD4B2+oeQhOynJVSP3Q2WwwwdkteR6YarrqGxHFcmwnz6GlRjTkCeHuqRRarMsNRHY0BkFAs5vRvmkDGDHiuUY9U3TKzA4mUtCwlNAePL80zQ5elrvSQDaa9n4CnfDFGnEhhox8n4NqB64wKIs3dDM7hfCD4YaRpVkavCPVYE/bcEWBFDYTzJOyucjXiWhVw15I9D0XcRZF38xD3egruc8mRqnjP4q8OB5a5aSh7pTTAd1kA/4XniFSbKBa0kYv82lF2ktNskYH1dCP3TpRxg+bBG5dK1Xs9eNgYPzlVBNW475Kk5xBl38pyNQKGbsAoyrRP2r1fRsyCLoOkTmvLw4yTHp3x4Hropr4bwL+fVKbBduczpPRCqIpH3WIDVmKzaJ4gPoowW0lPw2kN5UiDd4KrVMR8Os1Czn0L49wOgGl0KVc+u8FbvowNiKzQjZ3PkPNEeKyJ5j2+6gVevo+JVWE6QER116hanLYG2YCW3y4nmiu035xMoN/G7O7ZJ+tKu5wiX+/La/kYEdy7xLDbhqgSjzobfeCUIrAAw2R2V5JGDSC0ogKvyjzB/8lqdHKIMyPvTbmhIHpfIDLQAn5HVhp36IPE63ofx77/hf0rjibw5wq4wFmBFlfUgyeqs7YkWprbgP9yiYlsoOIgP1oJJM+EMFJ0vo2bPZEp33AuKIU5bDf2UxQ2Ify3HirPHiphuiHeCNETBzu8oW+yhE/Dw0FP/utDPLtoB3iFSaghmsuv+5/+DtCsklB9r2PxE7qkAFmJ9WojX8dgr6CxUFfZ7dBkS/09kWRube9My8X3aYtR30/02dUXEBGmijnSaEkb4P3k0EPFIRziETh4Dt8VMoh5CC2UzbWGO5BkzatkMO0Wh0C95DN2aM2m00I8TRChNbk17JM1YTZYmszMVHqZIGnu3ykSa9zAF57WEx79axSJJMEhMj3/xEVF4YDUJ7J7QKcT+wGqtOMIj9gm1S7vu/KEfsQ/MWngpSLu2f0KEb/FLQYLgorDIZuWtZauE1xglxFTto9PQ1xi9wj0wEel9DfVnI7x4rX3Ytk2kTNiL10JmI7wqsk2M21qDtv6rIhvMTHi5bWuZupXHabOX27byPIXh2rcGmuZY7Xt+gvRtpAEBWG2k+GQfVgBWslu4jeb3f6EJmcthjHP3AAAAAElFTkSuQmCC) at the right of the page, choose a name for the token, then click 'Create API token'. (You do not need to change the expiration date.)\n",
+    "5. Afterwards, you should see the table look something like this:"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "![image.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAsMAAABrCAYAAACSXI/oAAAgAElEQVR4Xu3dCVhN6QMG8DctSiZC9p0xzFgG2ZUlFMLIkkGWwYx9+BvZMxhjzSTTDJM9JoPIGMvIln2isa9DlsgMkZSi7f7POffe3K5KqZN7u+99nnlmps7yfb/vfPe85zvfORkphA/4oQAFKEABClCAAhSggAEKGDEMG2Crs8oUoAAFKEABClCAApIAwzAPBApQgAIUoAAFKEABgxVgGDbYpmfFKUABClCAAhSgAAUYhnkMUIACFKAABShAAQoYrADDsME2PStOAQpQgAIUoAAFKMAwzGOAAhSgAAUoQAEKUMBgBRiGDbbpWXEKUIACFKAABShAAYZhHgMUoAAFKEABClCAAgYrwDBssE3PilOAAhSgAAUoQAEKMAzzGKAABShAAQpQgAIUMFgBhmGDbXpWnAIUoAAFKEABClCAYZjHAAUoQAEKUIACFKCAwQowDBts07PiFKAABShAAQpQgAIMwzwGKEABClCAAhSgAAUMVoBh2GCbnhWnAAUoQAEKUIACFGAY5jFAAQpQgAIUoAAFKGCwAgzDBtv0rDgFKEABClCAAhSgAMMwjwEKUIACFKAABShAAYMVYBg22KZnxSlAAQpQgAIUoAAFGIZ5DFCAAhSgAAUoQAEKGKyALGH41vbJWHKxFeZ4dEQxDdpXIV6Y+FdLeI+xNVhwVlz3BcTjd96uyDQFNUIZuHw7Bx3Lv3v5Q9eNwuHiczDBWbNXZH97ydG7MX/GQ3RfPAQfm2V//ayuocAJLB16Dm1WjkS9rK7E5ShgAAJJD7Zgxsw/8TidujYfthJfNEkfgX3KAA4OVlEvBWQJw6fnNsbna56h7oAA/OZRB6Yqmritrmi9wxUhfi56icVCG4aAePwOP/8lAj0dNSpsjCKly8JKfTC/A8W7hmEFLsDDfhRK+hzFmDpAXoXhFPwKt6r7MCBsLTQl0qu6aDY3eT0CPWq+gwxXoYB+CSgSn+HhvzFIFop9wccFPimLsGJMNakSlsUroFih9OuTnT6lXyIsLQX0W0C2MPxFSA18fCcCLdYGYWx9Y0lJOwyLXyhnD+zElUgjWFWyRwe7ijBXeb54cg+vLMog9uLvOPJPAdRq2wUNbGKl5a9GlkQTF0dU1/jCibkbjH1Hw5FYvB46tauTo9Ci303K0udUQAx2o65NTveiTREXiQfPTFGqbBHVRV4CoiL+g6JoBVibCifIx0Yobn0f+wPOIrpQZdg7tUR51XH6KjoCUbBB6SKmUP+35bMT+DvqU7T6tAjS7w/C9h8exuyes1B81mYMbVRSWD8RkeHxKFyhuKq/JODJhf3Yc+EpTLSOf+mkLZSphHUUTuw9igcv3uw7ml6vos9h/85LQtlroE3Xm/imhmYYfr2fAhp1E+uyf143KRD8Mq5+qk1K/B0cUe2zQZe2qCXUmx8K5DeB9C4E1cf+/ReFUb2lE5pWtpCq/WYYjsPj8CdItiotfS8Ar/tYRudExc0gqa8Xqd4OTk1Lpw425TdX1ocCeSkgWxgWw0RAt83oNssG3qHL0FK4nasZhhUIwbTWo3D5077oZZuCgytW4b8WqxA4v5nUuf0GlsWSqw3QsEtXfHjXF2tDyqLex0CFxp2Bw944bDYJB7a6wUr48ji5oD0mBDWE66AGMLmyGb+GNIXv7qmy3kLOy0bivvJWILMwnJIYipmOQ/F8+EEs7W2N8K190OvHBlh/2B2VLnvAwTUIZpVqoH3X5nh5fB0CL9bFokPL0aEooHnSlPZxoCgKm5ZCuy8XYkqP8Az6QzkETJ+AnwOvwqx2M7TpNRuTexzSGLEVjv+5bTBsZyX0G9wZBf72EY7/TzBvny862QCvtMt0eBX+iBqIHXtHQXvGR+z52ejWdwcKt+8Pl4q3seXkFUSHVsD0sDXSyPCx2c3gfqwZXAc0wMujP2HThbbwDf4O1rsmY7L3TvyjqIvmLfrAY243FBa29dmwo6jvOgh1Cl9EwKq/4bBiT+qFcd62KPdGAfkEtMNwUqQ/BjjMQ0ybwehV6z9s/WUnzHtswsapdWCc5m6Lsu9OvjYWm1e7oqTpI2wc6og1sb3g6lwRUUIf2xv3DQKEO6nixCrxnOgT1gBlGzqgm9A/160LRvOFZ/GdYwH5KsctU8BABGQNwyF+ztgxug4WR8/DDqFDm2tNk0hMTISpqXK0SDxpO31ljhXHpqKGquOHdLiPZf0KQIFgjG/4AxrvDURf1QlevWyVu/Pg3DMac4/Ph61q/uSVJa0wO3kDNk2sYCDNyGrmpoByms/TNJs0UpTD2MBgaZpCUuQ69O8UhJ7r7bGx7wkM+3NtavB0HGaMn0/MRC1p7QRsHfExNpT5Q5o+oB2GpzzyxJ6ldqkjOxn1hw+1pkloji61FY7/Tp3CMfXsT2gjHf8J2D+pEebGe+OQdyupX2mWKePbtFFY7lofh22Ppvab5PiVcP3kOIapwrBC6K9JQn8Ve6z21A3NuikQhnmOrlBMP4Vpdsq7QolCObsONYdv0Pg3Qnhuth23RYG8FkgbhpV9fmvVQ6n9KOnJz+jZfC96HNiBfuWVU4/cwnxRZG5ruJ8bBr+Ng1BZ6LvPd38Bp5X2+GPbINWzNuK2miC0UyjmdSkghWH1OVGsI6cm5XVLc3/5WUDmMOwCcSRtqoMbYr86joUFh6fOGVbgEbaN7465wa9gYlEcH9e2xLXLtlivEYb/7hyBH3orT7wz7ZegxRHl3EXN4FxeCNgNFsahZd1Sqe2U8vwqrhacgJOcm5yfj13Z6pbZyLB6p+H+PeE04zrazDqCH/tZp3tBJ/5Q826IdhjWnGObWX/ILAzbpTMP/9X5yegwygq+Ql8SR6s1LzIzCsPa4VYsu/ay93ePQt/pJxFXoBBK1v4EZhevwmHtEekCQbNu4nr9qy3FizZ1hUkhCiWZ0TNcP/oBpl9TjjLzQ4H8IpD2QjDt/H5lHROwun8VXOz6AJ69hb5RdQ+qD7mNgM22+OW0F1qoBnHE7QwLqgbb6papNAkPQvG8mb90MS2GYfU5kWE4vxw9rIeuCMgehsWKxp73gLPrcTj2L4Pt112kuZjxQUNhv7A2tgeNk0aKYs9MROdx1tIJXD0ynNUw3NK/FQK8usBEQ9XIogTKl1DO0+KHAtkReFsYFoPrhoEO+CnMEmaV3bFddRtT++6GuM+I1c7ocewrnFzd5Y2RYc0wnFl/eFsYbhXgghP+rqkjzOK2WnnWx+/CVAjrbIXhEbBeehz/q6/U0gzD7bELw2t7o/HOPzG0ivC7xFOY1NodFZdnFIYD0WG/Jxw0OyUKwSZ1nnN2WoTLUkB3Bd4Mw2n7kQL3sLizI6LHXsVsRyEMV5+BO81Ho1WkD6433Z76kLm4nelP5mLl+LQPoZqq5hMzDOvuMcCS6b9AnoRhkenqMgd0XxYOqybzpTAcF+gG+/UOOCjcErISfq85mpWdMFwt2h9u9r5oqvGgXvzNq/ivYi3p1hM/FMiuwNvCsDgqPGh3X2zxs0Ngv044322fNH9YDMPtet/GxMN+6CpM51HOL+6Dx25/Y/ngDzINw5n1h8zCsEP0L+jd5De09t8nzccVg/qa/q1x0PZPbBhX4Y3pR5k9zb5rfA3Mj5wvTWkS5yg+Ey5Qu/SOlOYMt8dmDP5wC5xDtqCXMBCugHDHp8V4lE0nDEN4TNBvYHNsr7Y59USvSLyBG3eq4qMP06Tj7DYNl6eAzgloT1c4Mq0ept2bmdqPYoU7NZ36v8D3wrMzzc1ev6HFQTW32GZacOr3R/vP72HyAeW0K/Hz39WrMK1VK3XOMEeGda75WaB8IpBnYVg5mtYKy1LmSmFYDAqePYZgy6s6qFuhIBJe3hFOlm2zPU1CDM4Pgr/B4DH7kFyjCUrFX8DN2EaY/euPcCqXT1qJ1chTgczmDA9O8UCXwfcwUTVPWHxYZmAHXzRZFYSvzGahw5dhqFUuDGfCEpAcH4siTb7HttXKcJnZNInM+0OCNPd+8hErVO24GDsXPEjzyjPx+B8wcg+iLCxglJCAUm0WYsNSJ2mf2qPVmYXh5MgDmNR3OHb9WxSWZgqUammLAr+/wmhpznACTi3rglFrjPBxg/IomPgSdy7dQzfVNInnwmh0yzEhKFTcGV7Hv0cjYVuT3cbiQPzHqF/+BS5fjkNrj02Y3710nrYld0YBuQW0w7B44ffzoJ746WJBmJslIcmoKsb8shlDhItV7f6nvmvqIl3MJuPS6gEY7BWGMnXrweT+aTwtMQBL1/0P9YSZExwZlrsluX1DFpAlDGcHVHyFWoxJGdVrZbKzpvayyldcvbJUv6ImJ9viuhTIvoBm8Cz3jsd1xv0hTut1atrlU76iyTiTd5xmtUaZ9ck3Xy33eqvK1yFWTPOOVfG1a49fWGq8ii6rpeByFNBvgXc/9nOvL+u3IEtPgbwTeO9hOO+qyj1RQF6B9OYMy7tHbp0CFKAABShAgZwKMAznVJDrU0AlkPzsLHYfMUarrnWlefD8UIACFKAABSig+wIMw7rfRiwhBShAAQpQgAIUoIBMAgzDMsFysxSgAAUoQAEKUIACui/AMKz7bcQSUoACFKAABShAAQrIJMAwLBMsN0sBClCAAhSgAAUooPsCDMO630YsIQUoQAEKUIACFKCATAIMwzLBcrMUoAAFKEABClCAArovwDCs+23EElKAAhSgAAUoQAEKyCTAMCwTLDdLAQpQgAIUoAAFKKD7AgzDut9GLCEFKEABClCAAhSggEwCDMMywXKzFKAABShAAQpQgAK6L5BhGH769Knul54lpAAFKEABClCAAhSgQA4EODKcAzyuSgEKUIACFKAABSig3wIMw/rdfiw9BShAAQpQgAIUoEAOBBiGc4DHVSlAAQpQgAIUoAAF9FuAYVi/24+lpwAFKEABClCAAhTIgQDDcA7wuCoFKEABClCAAhSggH4LMAzrd/ux9BSgAAUoQAEKUIACORBgGM4BHlelAAUoQAEKUIACFNBvAYZh/W4/lp4CFKAABShAAQpQIAcCDMM5wOOqFKAABShAAQpQgAL6LcAwrN/tx9JTgAIUoAAFKEABCuRAgGE4B3hclQIUoAAFKEABClBAvwUYhvW7/Vh6ClCAAhSgAAUoQIEcCDAM5wCPq1KAAhSgAAUoQAEK6LcAw7B+tx9LTwEKUIACFKAABSiQAwGG4RzgcVUKUIACFKAABShAAf0WYBjW7/Zj6SlAAQpQgAIUoAAFciDAMJwDPK5KAQpQgAIUoAAFKKDfAgzD+t1+LD0FKEABClCAAhSgQA4EGIZzgMdVKUABClCAAhSgAAX0W4BhWL/bj6WnAAUoQAEKUIACsgnEx8fj5cuX2dq+ubk5LCwssrXO+1yYYfh96nPfFKAABShAAQpQQIcFbt26BfGf7HyqVasG8R99+TAM60tLsZwUoAAFKEABClAgjwXUYbhRo0ZZ2vPp06elIMwwnCWurC4Uh8fhz1CwdFlYmWZ1HS5HAQpQgAIUyH8CN4+ux6m7BdKtWLUW/dGsivJXisRn+OevAwi5Ey/9f+k6XdGqnhV4Gs1/x4QcNRKnRkRERKBs2bLSv8VA3KFDhyztat++fe8UhqOiomBtbZ3hPsQyyTX1Is9Ghl9d9kC7LuGYHrYGjlniBJIi/THE8VuEKixRu+dGbJpaK4trpr+YAhfg0aorYkfdwQ+9lV8WD/9NRrEKxWGeoy1z5fwqcGv7ZCy93xPeY2zfWkVx2d9SvsbUHmXeWDY58gA85xxCnYnfoWN55a8jghfAc19NTJzbDaUz2boi8QGOrvKGV8B+hEcVw0ftBuJbj/6oXuj1SvEP/4TXpAUIuPIcJWv1w5QFY2BX1kS1QALu/eWLZYu34dDtaJSo7Ighc2agVy3lUZ8SfxlrZk/Hr0HhiDEvjSaf/Q/Tv26LUjxrvrXNuQAF8lpg67g6mH+soLDbZMQ/f44U82KwNFNIxejkcQazuwKx5z3R94vluGX8ERrULY2CRlG4cfIykquPhu/m0fjYLK9Lzf3pm4AYTMURXnE0+OnTp7KHYfX+xPBdu3btN7hiYmJw5swZVKxYUZYRZ50Ow2cXNMeXp0fgwFY3WOXCkaQdht8loOdCMbgJPRI4PbcxRl2bjBA/l7eWWlx2bvJ6BHrUTLNsSvwpTOvshm33y2J0QDDG1AGenZmCbv0D8KzEIGw5NhU1Mtx6DDYOboTVRv/DHPdOKG/5EH/OGoHl9wdjx95REHN1cvQWfGE/B6bD12GmswVC14/H7E3V4R26DC2Fk97jbX3RYaElxi6aiHaVLXD/8DSM/+4Jhu/dhYFVzmKa3QDcajkHU0Y2hHXcBayaOAlHyi9F0E8OHEV6a6tzAQq8HwHt85m6FOL3wWD7qYjp+hv85zRIHehJSQzFTMfeOFxuEXYI32fF3k+xuVc9EcjrMCyyXLp0KXU0WjMQq4Ow+FCeGM5NTNQDPbmHKWMYjsOtY3twUrhFI96eaWayGE7pjAzH3A3GvqPheFWoMuydWqK8NNqVgKiI/3DC2wU+KYuwQkgPNqrRW3E09+yBnbgSaQSrSvboYFcxtbMr4iLx4JkpSpUtknoSfxUdgceJ1ihfwgKaXx5LnCMRdup7uA19iFHBC+FgVRqli3AoLPcOrfyxpbRhOA6R4fEonHonIQHPIiJRwEY5hSe9MKzAv1jeuw3215+JGnt8UNbnKEaU+gW9221FsylNsNvHAisyDcPCHYy4OKF/FHp9Uktai88/2oKOwbswSEjDV5a0wvDrU3BghZPquI/Cctf6ONbsFDaME8ec4xAfVwgWqSPJCVjdvwoO2oZKv09MTISp6etj/9X5yegwygq+bylX/mhh1oIC8gqId4VmjZyCgHPPYFLsU7jN74jbEy6gZ+gPaCPtOgGX/L8SLlZDcD/WCEUrtsSUn39E9w9NEHfTD6NHeOKvu3EoYFkJzfvNwLxv7KUgm1EYFr8PPt/THbuCxkkXy5qfxLvz4NQ2EI7+f8G9sbz15tb1W+B9hOH0AnFeBGFxv7KEYfEKdE7nPvB/UB71mlWH8cML+Ne4OJ5fKoW5qmkSCjzCttEdMeOEDRo2KA+zl/dw9iwwZMMejKx/HYu694P/jRfCabwwipq3wnzhi6ORdOtnNWKqNEc16zjc/TsEkdYj8dv+8dLIWtxWVzTxqocAjZO4ZpjR/PL43nIE7GYcwfPnybAoWhif9tuEdeOr6/fRy9LnuoDm8ZOCX+FWdR8GhK2VpvpIx5P9KJQUAq442pvRyLBCCJuJplcxR2NZMYCm3JgDp6/M3xqGtSslntA6tbuBCZfXwMksCj496+Ja97tY1u/1PMKI1c7oFuSGE/6ub4zuKhCGeY7t8GjITXj1Tjv3ULzYPDTLCbOfzuXIcK4fTdygoQmI3xGz2/ZAaEMf/DSrA2wSz2G9+wj8EFQTP6jOhfeEvuroVRhT/HzxeT1j3Nk7E95/fY4FM23g07kD7vQ+DK+BpWAcfwfrPH9HE/ex0jSH9MOw8kL39xoH3rhDJdorcEUoTyf801V9oWxoLcL6ZlXgfYVhzUBcsmRJiOWQc0RY7SFLGBavTHv622J1sCeaqkajwv17ovO0D7BI9QXwcGsfOC+tBb8DM1PnL4ULP3PxqouNqjCrHS7EkB1y9hM0baya65gYiGH156Daz6GYape9MCzOGeY0iax2C8NdLjfCsPIklDY4iz8Tj7/shmHxInJNfztsLr0OOxc3hYnWPHh1S6V3Yaj+ndgXuy6pDt/j82GrmjsolqVtt434Txh/ruo0D7/92BUZP8ZguMcDa06B7Ai8PP417L5MxMKzP6GNqq9pXlR3kL4X+iBp4iXM66J1YYp7WNy5Nc7YboDnpOaqu6av955eGFb/7KLD3nTDsLi238CyCKh2MMPfZ6d+XDb/CrzPMCyqnjt3Do8ePZLuWtrZ2ckyNUKz9WQIw8qRqqC6QWk6W9rgmfY2rbpACgRjfMNZ+GjjQYwQpl2+OdKmfBDIx/csom0+Qc8BLRA8vHfqA3HZGRlmGM6/nTg3a6ZLYVh8kG7t0HZY8WQ0Vm8bpRodUo70PBuufChUMwzb/doRh7YN0phvn4Aw/wFwmRePMeu2YUh94zeoXkXfwu7vB2HpY3f8vrpLrszVz8324LYooE8C6Z2TNMNwe+FuU/9qK9AsUPksgfZHnCbh/vWPOPRPFIyL1UCXMd74tl9V6W5PRtMk/IdWwJqiW7BPuFhWf5IjQ3A2qgEafngeU1r2xIsRt9LcSdInU5Y1bwTeZxhWT41QKBRISkqS3miR3kN1uSmR62E4ow6qGYalq2HhrQ5bo2xSn4JVV8oIlui1JBju9m+G4avLHODia4OxPrPRuXI0Tqz0wrKAU2jscUsKAgzDuXlocFuigK6EYfEhvPmuQ/FHwW/w68ZBqKzxNPiO0dWxpuQfaS4+90/6BJ5Gftgzv4GqIeNwdIkLxqwtiskBG9BHmI+Y0SdFnJNc4zC+UE0H4ZFAAQq8m0B80FC0mFkBG0/MhPpdSGnD8C4Mr+2Byqq7mxnvJQ7/hq7GyC98UXfJeXzrkHEYjljXDQ4Li8Hz+Bp0Kqrc4jnvtui3siKmTimIRR7RmBGyCT146+fdGtVA1npfYVh7jvC1a9fSfagut5sh18Ow+DCAOGdpU+mANFem2lMS0rt61a6c5siwQrplZIcbn12C77APpEW1gzfDcG4fHoa3PXEqzqHD5WDfvjRMhJFY789b4GSTo9g0sQLEk1j/alvgfGo7+tqIr/7bhqGOXmi49kimc4ZTj1WNOcPiz9JOk9B+OO+1ffzDAHztMgORrVZg5Xy7N54Cl/pW79sYF7QGPYTXqcWe94Bz71AMEt4WMUh456g4orx5fFcsvNYRP/p/h2ZC2dWflLhTmDFkA5rPWyxcYIrTj4TRY7++6OFdE+tOf4e6hncIsMYUyJLAwzMB+KdIN9hLD7oF4aAw8urcqLhwXnqEv7ZdQmnntqhotBvDbSfD3D0Inv1KCSO6qrsz0yywQDVl8Mi0enC/OhqbNg2TLnLF/r79r5bo9Zk1TuzYjZqdPpNecyhud+lnTfBgwG0sEl5uk9HAk7jchoGt4PmgB5at/1b1isU4bBtlC/c/E1B7QNoL5yxVlgsZnMD7CMMZPSyX0VsmcrNRZAjDgDgfuP3Uxxi8YTcmNC4onYx3Te+DiVuqw0v1BRBzfBwcBl+Eq2oZsVLSl0BoG/R1Vr70Je00iQRhrlN1bLTZLM2VFG8TpcTvxUi7kbBwD1OODAe6wVb4kll6+he0F+Yqi+9endJvJE4Ir5IRX42V/qvVbmPSNT84GqV9oj43kbkt/RFIitwhBM+JOPisCMyTY1DoQ833ckbB/ys7zD5mBkuLAvigSlOUiriAFstzHoYrX5+N9p23wM7nIr7TeBG3+oEXv3tpR3KNFOUwNvXWqvA0+uq+cFt4DUaFCyA+tih6zt8Cj8/Eky8gvqKwl++jNxpBeVKsits7xmPgtAN4alYYBZNiEWfyCb5ZtTndaRT605IsKQXkE1D3y72Vl+GkMJ1IHNyZd3MMAg+7o7zUlzeg8cIbUmh9EPwN3L76HY8KWcFcYYqag9sj0ete6p0XReIN/DykD7xDklC4sDFexn+A7nPF/muE38d3h8e+GBSvXAEmD8ORVPFLLFe9JzijMCxdfAvbXDv6Cyw+9FB6D7FJwlMkGlfEJzXiEBbTA7/tdkcNvmtYvgMkH2w5vTBcrFjWXsgnvpc4u3+BTvyDGqdOncrwYTl1IM7udrPaFLKEYfHK9M9pvTHutwiYWFnBOKkQ2g1sgBCfaI0/upEgnYT7TzqA2MKFpSvml/HFhS+BTaknce05w4l3V6KfyyLcKdEYdWye47GiKizu7kDZscowrH6Lhd+dgrAS3nVlYtkGPRqeQsDjb9INwymJuzG66dc4lPgBSjacir1resIiq3JcLl8LvHhyDzEmZdJ93V5mv3t3lATsHT8E/07wk16X9i4f5R+REd6/8k5/rVH5OsMXZiWk1xDyQwEKZC6giItGjGkR6bWKYt+LSSwKK9UD43HR0TAt8voVn+LrDR+HP4Fx8QooWijtW2nUexFfDXr/SQKKaPVfZb+OQYF3ev2ncr+vLJT9Wjw3X79qjJq1irN5KZCpgGYYjhNe7/nw4cNsiZUpUwblypXL1jriX7mrVKlShg/Lve332dqZ1sKyhOG3de60ZVB1VmOrNO8HzrhSyuWTM/xiyN5JPWcBIif0XJcCrwWSHgdizb66GKR6OIY2FKBA/hTQfkVj/qwla6XvApphOLM/kazv9VSXX9YwnF+QWA8KUIACFKBAbggocAJLh55Dm5UjUS83NshtUEAGAXUYzs6mxWkUtra22VlFZ5ZlGNaZpmBBKEABClCAAhSgwPsXEOfwRkREZKsg4h/HyO7UiGztQMaFGYZlxOWmKUABClCAAhSgAAV0W4BhWLfbh6WjAAUoQAEKUIACFJBRgGFYRlxumgIUoAAFKEABClBAtwUYhnW7fVg6ClCAAhSgAAUoQAEZBRiGZcTlpilAAQpQgAIUoAAFdFuAYVi324elowAFKEABClCAAhSQUYBhWEZcbpoCFKAABShAAQpQQLcFGIZ1u31YOgpQgAIUoAAFKEABGQUYhmXE5aYpQAEKUIACFKAABXRbgGFYt9uHpaMABShAAQpQgAIUkFGAYVhGXG6aAhSgAAUoQAEKUEC3BRiGdbt9WDoKUIACFKAABShAARkFGIZlxOWmKUABClCAAhSgAAV0W4BhWLfbh6WjAAUoQAEKUIACFJBRgGFYRlxumgIUoAAFKEABClBAtwUYhnW7fVg6ClCAAhSgAAUoQAEZBRiGZcTlpilAAQpQgAIUoAAFdFuAYVi324elowAFKEABClCAAhSQUYBhWEZcbpoCFKAABShAAQpQQLcFGIZ1u31YOgpQgL34LDMAAACaSURBVAIUoAAFKEABGQUYhmXE5aYpQAEKUIACFKAABXRbgGFYt9uHpaMABShAAQpQgAIUkFGAYVhGXG6aAhSgAAUoQAEKUEC3BRiGdbt9WDoKUIACFKAABShAARkFGIZlxOWmKUABClCAAhSgAAV0W4BhWLfbh6WjAAUoQAEKUIACFJBRgGFYRlxumgIUoAAFKEABClBAtwX+D4/nW2ut3arDAAAAAElFTkSuQmCC)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "6. Click the two overlaid squares ![image.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEEAAAA/CAYAAAC/36X0AAABf0lEQVR4Xu2aPYqFMBhFvwcWKlhoqaWFG9DNuwVXYGGprYUgdjMRMTgyPHRiRpk5AeEJ5u+8e6+R5PWhivzz8gKCCBCUC4AAhCUMUQIQUIJeGGAH7IAdsMP2S4FMuDoT+r6Xqqoe8TlWFIWEYXhoLJcqAQiKORC+geC6rvi+f0iSpg+N4yjTNOlmHmOHNE1lvn6jNE0j87UWICgSQLANYQ6/tTiOI0EQ6Pt9MP5ZO5RlqScdRZHkeQ4EIKAEEeygkgAIQFheCCgBCCiBxRIrRqUBIABhiQKUAASUoF+L2AE7YAfssN03IBPIhPOZEMexJEmyFZK1323bStd1un2rmy9n7GBtxgcaBoKCBIQ7IQzDIHVdHxCr/UeyLPuyRfiux9MnVd5lgv2p2ekBCIorEEwheJ4n81rgjjIfBbpqDWKkhDsmv/a53/gxGQsQTO1gQt+07q1K2B7XMZ2ISf39USGTtk7bwaSzp9YFwk8y4an/psm4UAJKWPSDEoCwKOET7bDhYssL2DsAAAAASUVORK5CYII=) to copy the API key; then run the code below and paste it into the input text box and press Enter."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "os.environ['ACTIVELOOP_TOKEN'] = getpass(\"DeepLake API key: \")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Finally, pick a name for your dataset. It doesn't matter what this is, but keep in mind that it will be shared with the students."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dataset_name = input(\"Enter a name for your dataset: \")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Processing The Document(s)\n",
+    "\n",
+    "In this part, you will upload the documents you want the students / model to reference; the embeddings will be created from those documents.\n",
+    "\n",
+    "**Note: The embeddings of all the documents you share will be publicly available. Do not use this for any documents you want to keep private.**"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "First, upload your documents to Google Colab. To do this:\n",
+    "1. Click on the 'file' icon ![image1_3.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAFQAAABUCAYAAAAcaxDBAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAALEoAACxKAXd6dE0AAAIKSURBVHhe7do/kmFRFMfxM7MAGyCRKZEI60CA0CJkEjKJHZApAevwJxJhAWyABfTM6T7JdDZ1v6/rvfL7VHV1ndfJ7W9f+ql3f338ZYL5Hd8FoqAwBYUpKExBYQoKU1CYgsIUFKagMAWFKShMQWEKClNQmILCFBSmoDAFhSkoTEFhCgpTUJiCwrCDDs/n01arlW23W3s8HnH1/83nc+v1ejEVDxLUYw4GA7ter3ElTZGjIi/56XSKxXTj8fhzpxdR8g713dloNGJiFXGnJgc9HA42HA5j4k0mE6vX6zExyuWyVSqVmFi5D5qVVquVyR/rbW+bjsfj5z/Sy+USVxhvfR/6er1sNpvFxHj7G3vfqff7PaZ0mbyHNptNa7fbMeWLr/d0OsX0Zb1ec+v1oCn2+/1HtVr952uxWMRP88fX9n29/jtQ9FkepqAwBYUpKExBYQoKU1CYgsIUFKagMAWFKShMQWEKClNQmILCFBSmoDAFhSkoTEFhmTxGzvLsUCp/Bv/9/Cr5GDnXp+9+yvl8tlKpFFOa5Je8L6TT6cRUPL52KqZL3qHOd2m/37fb7RZXiqFWq9lms8lfUOdRl8ul7Xa7pDP2P8Hf47vdro1GIzSmw4LKF902wRQUpqAwBYUpKExBYQoKU1CYgsIUFKagMAWFKShMQWEKClNQmILCFBSmoDAFhSkoyuwPHzLZbD4ZgJMAAAAASUVORK5CYII=) at the bottom of the sidebar to the left of these instructions.\n",
+    "2. Click on the 'upload file' icon ![image2_1.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAFcAAABaCAYAAADNVsqyAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAALEoAACxKAXd6dE0AAAKKSURBVHhe7dw7bsJAFIVhJx3UtLARaNkILIKakkXAGhAlNSyCmj3QJjnRjOQgY173eO5kzi+h2Fgok08Tg8zAx9dPlaL0GX4qQsIlJlxiwiUmXGLCJSZcYsIlJlxiwiUmXGLCJSZcYsIlJlxiwiUmXGLCJUZ9m+d4PFb7/b46n8/hHm7j8biaz+dhL30U3MvlUq1Wq85Q63kCppwWUsEi/Les1+uwlzZz3MPhkAw25gXYHBd/mIc8AJvjnk6nsJW+1MDmT2iz2Sxs/W2z2YQt2279vnqpnuSKeJ2bagYXgYtSABeDi7oGLgoXdQlcHC7qCrhIXNQFcLG4iA1cNC5iAhePi1jAwg0xgIVbyxpYuFcB2CrhEhMusewvOb4Te6yaucSES0y4xIRLTLjEssLFmgjcmFeyLMsGF6h4iYQb80qWZVngRth6OQC7x22CjXkHdo3bBhvzDOwW9xHYGIC3223Y85NL3GdgY7vd7vdxnnKH+wpsDI/zBOwK9x3YmCdgN7gWsDEvwC5wLWFjHoCT496D7fV6Yau5tuOpgZPjtq1EB9xisQh7zeF4G3DKle7JcbHiGyu/r4uwo9Eo3NMcjt8CTv2xKRfn3Gvg4XBYLZfLu7CxJmAPn0dzgYsiMGABNRgMwpHHqgN7gEVucBFAANTv98M9zwVgzHgPsMgVLnoVNvbsjGfmDvc/JVxiwiUmXGLCJSZcYlrl2JDVWDVziQmXWGenhZxye1rAhZecsxy/Oe50Og1beWY5fnPcyWSS7ezFuDF+qyhPaLhsmBswxotxW0b5RrwYvmoQS41Sf89YW0DFqcByxsaouKWn17nEhEtMuMSES0y4xIRLTLjEhEtMuMSES6uqvgFS+TQXb05HUQAAAABJRU5ErkJggg==) on the left of the Files toolbar.\n",
+    "3. Select all of the files you want to upload, then click 'Open'.\n",
+    "4. A warning should pop up. Click 'OK' to continue.\n",
+    "5. Wait until the spinning circle in the bottom of the 'Files' section disappears. This means that all of the files have been uploaded."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Adding YouTube Videos / Websites\n",
+    "If you have any websites or YouTube videos which also contain content which you want to put into your data lake, paste those links one at a time into the text box below, pressing 'Enter' after each one. Once you have entered all the links, press 'Enter' without typing anything to finish execution of the code cell.\n",
+    "\n",
+    "If you have no URLs to add, just click on the box and press 'Enter' without typing anything."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "url_list = []\n",
+    "while (url := input(\"Enter a YouTube / website link: \")): url_list.append(url)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Model for embeddings\n",
+    "\n",
+    "Below, you can choose a different model which will be used to create the embeddings. At the current time, only OpenAI models are supported. If you're not sure, the following setting should suffice."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model_name = 'text-embedding-ada-002'"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Embedding & Database Creation\n",
+    "\n",
+    "Now that you've made all of the relevant settings, click the \"Run\" arrow next to this code block, or select this cell and then click \"Run This Cell and All Below\" or \"Run All Below\". This will automatically execute the rest of the code so that your database can be created from your specifications.\n",
+    "\n",
+    "You can ignore any warnings that pop up, but if the code stops execution, read the error. If you cannot fix it, please contact the developer."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Library download and installation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# run this code if you're using Google Colab or don't have these packages installed in your computing environment\n",
+    "#! pip install git+https://<token>@github.com/vanderbilt-data-science/lo-achievement.git\n",
+    "#! pip install deeplake"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# basic libraries\n",
+    "import os\n",
+    "from getpass import getpass\n",
+    "from IPython.display import display, Markdown\n",
+    "\n",
+    "# libraries from our package\n",
+    "from ai_classroom_suite.PromptInteractionBase import *\n",
+    "from ai_classroom_suite.MediaVectorStores import *\n",
+    "\n",
+    "# from langchain\n",
+    "import deeplake\n",
+    "from langchain.vectorstores import DeepLake\n",
+    "from langchain.embeddings import OpenAIEmbeddings"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#setup OpenAI API key\n",
+    "os.environ[\"OPENAI_API_KEY\"] = OPENAI_API_KEY\n",
+    "openai.api_key = OPENAI_API_KEY"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# get transcripts from youtube URLs\n",
+    "yt_docs, yt_save_path = get_website_youtube_text_file(url_list)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Now, we'll create the embeddings and the vector store from the transcripts of the YouTube videos. Make sure that all your documents are shown in the output from the previous code cell, then continue execution."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# create document segments\n",
+    "doc_segments = rawtext_to_doc_split(yt_docs)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Make sure that all of your documents are shown in the output from the previous code cell, then continue execution."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# create embeddings\n",
+    "embeddings = OpenAIEmbeddings(model=model_name)\n",
+    "\n",
+    "### Dataset Creation ###\n",
+    "dataset_path = f\"hub://{DEEPLAKE_USERNAME}/{dataset_name}\"\n",
+    "db = DeepLake.from_documents(all_document_segments, dataset_path=dataset_path,\n",
+    "                             embedding=embeddings, public=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Sharing With Students"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "display(Markdown(f'''To let students access the repository, give them the following URL:\n",
+    "\n",
+    "`{dataset_path}`'''))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Distribute the URL above to students. They will copy and paste it into the LLM learning application, which then allows their models to use all of the documents you uploaded as reference sources when responding to or creating questions."
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "python3",
+   "name": "python3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}

lo-achievement/nbs/_quarto.yml

@@ -0,0 +1,20 @@
+project:
+  type: website
+
+format:
+  html:
+    theme: cosmo
+    css: styles.css
+    toc: true
+
+website:
+  twitter-card: true
+  open-graph: true
+  repo-actions: [issue]
+  navbar:
+    background: primary
+    search: true
+  sidebar:
+    style: floating
+
+metadata-files: [nbdev.yml, sidebar.yml]

lo-achievement/nbs/helper_utilities.ipynb

@@ -0,0 +1,405 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# helper_utilities.ipynb\n",
+    "> Helper functions for when we need to work with files in Google Colab or locally\n",
+    "\n",
+    "In this notebook, we write some generic code to help us interface more easily with loading in files."
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "skip_exec: true\n",
+    "---"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#| default_exp IOHelperUtilities"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#| export\n",
+    "import ipywidgets as widgets\n",
+    "from IPython.display import display, clear_output\n",
+    "from functools import partial\n",
+    "from ipyfilechooser import FileChooser\n",
+    "import os"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#| export\n",
+    "def check_is_colab():\n",
+    "    \"\"\"\n",
+    "    Check if the current environment is Google Colab.\n",
+    "    \"\"\"\n",
+    "    try:\n",
+    "        import google.colab\n",
+    "        return True\n",
+    "    except:\n",
+    "        return False"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "assert not check_is_colab(), 'On this system, we should not be in Colab'"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## File Choosers\n",
+    "Jupyter notebooks, the different IDEs, and ipywidgets currently (as of August 8, 2023) are not playing nice together, and also are misaligned in terms of versions. What works in Jupyter Lab at version 8 somehow doesn't work in Google Colab and changes are needed. Neither the Google Colab version or the Jupyter Lab version work with VSCode.\n",
+    "\n",
+    "While this is being worked out between VS Code developers and ipywidgets, we've found a mid-term solutions which requires another package. We implement and test this below (thanks, Code Interpreter!)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#| export\n",
+    "class MultiFileChooser:\n",
+    "    def __init__(self):\n",
+    "        self.fc = FileChooser('.')\n",
+    "        self.fc.title = \"Use the following file chooser to add each file individually.\\n You can remove files by clicking the remove button.\"\n",
+    "        self.fc.use_dir_icons = True\n",
+    "        self.fc.show_only_dirs = False\n",
+    "        self.selected_files = []\n",
+    "        \n",
+    "        self.fc.register_callback(self.file_selected)\n",
+    "        \n",
+    "        self.output = widgets.Output()\n",
+    "        \n",
+    "    def file_selected(self, chooser):\n",
+    "        if self.fc.selected is not None and self.fc.selected not in self.selected_files:\n",
+    "            self.selected_files.append(self.fc.selected)\n",
+    "            self.update_display()\n",
+    "    \n",
+    "    def update_display(self):\n",
+    "        with self.output:\n",
+    "            clear_output()\n",
+    "            for this_file in self.selected_files:\n",
+    "                remove_button = widgets.Button(description=\"Remove\", tooltip=\"Remove this file\")\n",
+    "                remove_button.on_click(partial(self.remove_file, file=this_file))\n",
+    "                display(widgets.HBox([widgets.Label(value=this_file), remove_button]))\n",
+    "    \n",
+    "    def remove_file(self, button, this_file):\n",
+    "        if this_file in self.selected_files:\n",
+    "            self.selected_files.remove(this_file)\n",
+    "            self.update_display()\n",
+    "    \n",
+    "    def display(self):\n",
+    "        display(self.fc, self.output)\n",
+    "    \n",
+    "    def get_selected_files(self):\n",
+    "        return self.selected_files"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Now we test the file chooser very briefly to ensure that the results are as we desire."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "07ff01c3633c4addb72bfe758fb9c4e4",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "FileChooser(path='/workspaces/lo-achievement/nbs', filename='', title='Use the following file chooser to add e…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "1aa36e43d7254a2f8669a92e156b726c",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Output()"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "# Create file chooser and interact\n",
+    "mfc = MultiFileChooser()\n",
+    "mfc.display()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['/workspaces/lo-achievement/nbs/_quarto.yml',\n",
+       " '/workspaces/lo-achievement/nbs/nbdev.yml']"
+      ]
+     },
+     "execution_count": null,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# get files that were selected.\n",
+    "mfc.get_selected_files()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## File loading\n",
+    "Now, we implement a file chooser that will work across platforms, whether it be Google Colab or local environments."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#| export\n",
+    "def setup_drives(upload_set):\n",
+    "\n",
+    "    upload_set = upload_set.lower()\n",
+    "    uploaded = None\n",
+    "\n",
+    "    # allow them to mount the drive if they chose Google Colab.\n",
+    "    if upload_set == 'google drive':\n",
+    "        if check_is_colab():\n",
+    "            from google.colab import drive\n",
+    "            drive.mount('/content/drive')\n",
+    "        else:\n",
+    "            raise ValueError(\"It looks like you're not on Google Colab. Google Drive mounting is currently only implemented for Google Colab.\")\n",
+    "\n",
+    "    # Everything else means that they'll need to use a file chooser (including Google Drive)\n",
+    "    if check_is_colab():\n",
+    "        from google.colab import files\n",
+    "        uploaded = files.upload()\n",
+    "    else:\n",
+    "        # Create file chooser and interact\n",
+    "        mfc = MultiFileChooser()\n",
+    "        mfc.display()\n",
+    "        uploaded = mfc.get_selected_files()\n",
+    "    \n",
+    "    return uploaded"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "c88d8353fb0d4dc6a2df946ea2082e5f",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "FileChooser(path='/workspaces/lo-achievement/nbs', filename='', title='Use the following file chooser to add e…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "9b8ed072582b4e87ace35d2d59c3a82f",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Output()"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "res = setup_drives('local drive')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['/workspaces/lo-achievement/nbs/_quarto.yml',\n",
+       " '/workspaces/lo-achievement/nbs/nbdev.yml']"
+      ]
+     },
+     "execution_count": null,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "res"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Now, we'll verify the behavior of Google Drive. We'll wrap this in a try/except block so the code can run all the way through."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "An exception of type ValueError occurred. Arguments:\n",
+      "It looks like you're not on Google Colab. Google Drive mounting is currently only implemented for Google Colab.\n"
+     ]
+    }
+   ],
+   "source": [
+    "try:\n",
+    "    setup_drives('google drive')\n",
+    "except Exception as e:\n",
+    "    print(f\"An exception of type {type(e).__name__} occurred. Arguments:\\n{e}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Future expected implementation\n",
+    "\n",
+    "The following code is included as it works, just not in Visual Studio code. The current implementation of the File chooser is a bit inelegant, but this is due to the current limitations of the combination of the libraries and platforms. Once some errors with VS code can be updated, this code will be the preferable solution as it is more familiar to users."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import ipywidgets as widgets\n",
+    "from IPython.display import display\n",
+    "\n",
+    "class UniversalFileUpload:\n",
+    "\n",
+    "    def __init__(self):\n",
+    "        self.filelist = []\n",
+    "        self.uploader = None\n",
+    "        self.status_output = None\n",
+    "    \n",
+    "    def _process_upload(self, change):\n",
+    "        self.status_output.clear_output()\n",
+    "        with self.status_output:\n",
+    "            print('What is happening?')\n",
+    "            print(change)\n",
+    "\n",
+    "    def process_uploads(self, change):\n",
+    "        if change['new'] and change['new'] != None:\n",
+    "          with self.status_output:\n",
+    "              print(change)\n",
+    "          \n",
+    "          self.filelist = change['new']\n",
+    "          \n",
+    "          #get filenames and promt\n",
+    "          fnames = [fileinfo['name'] for fileinfo in self.filelist['metadata']]\n",
+    "          with self.status_output:\n",
+    "              print('Uploaded files:', fnames)\n",
+    "          \n",
+    "          #clear it so it doesn't save state\n",
+    "          self.uploader.close()\n",
+    "    \n",
+    "    def get_upload_value(self):\n",
+    "        return self.filelist\n",
+    "    \n",
+    "    def choose_files(self):\n",
+    "        self.uploader = widgets.FileUpload(accept='', multiple=True, description='cat')\n",
+    "        self.status_output = widgets.Output()\n",
+    "        self.file_output_box = widgets.VBox([self.uploader, self.status_output])\n",
+    "        self.uploader.observe(self._process_upload)\n",
+    "\n",
+    "        with self.status_output:\n",
+    "            print('Waiting...')\n",
+    "\n",
+    "        return self.file_output_box"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#test\n",
+    "ul = UniversalFileUpload()\n",
+    "ul.choose_files()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "python3",
+   "language": "python",
+   "name": "python3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

lo-achievement/nbs/media_stores.ipynb

@@ -0,0 +1,920 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# media_stores.ipynb\n",
+    "> A notebook for storing all types of media as vector stores\n",
+    "\n",
+    "In this notebook, we'll implement the functionality required to interact with many types of media stores. This is - not just for text files and pdfs, but also for images, audio, and video.\n",
+    "\n",
+    "Below are some references for integration of different media types into vector stores.\n",
+    "\n",
+    "- YouTube: https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/youtube_audio\n",
+    "- Websites:\n",
+    "  - https://js.langchain.com/docs/modules/indexes/document_loaders/examples/web_loaders/\n",
+    "  - https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/web_base\n",
+    "  - Extracting relevant information from website: https://www.oncrawl.com/technical-seo/extract-relevant-text-content-from-html-page/\n",
+    "\n",
+    ":::{.callout-caution}\n",
+    "These notebooks are development notebooks, meaning that they are meant to be run locally or somewhere that supports navigating a full repository (in other words, not Google Colab unless you clone the entire repository to drive and then mount the Drive-Repository.) However, it is expected if you're able to do all of those steps, you're likely also able to figure out the required pip installs for development there.\n",
+    ":::\n"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "skip_exec: true\n",
+    "---"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#| default_exp MediaVectorStores"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#| export\n",
+    "# import libraries here\n",
+    "import os\n",
+    "import itertools\n",
+    "\n",
+    "from langchain.embeddings import OpenAIEmbeddings\n",
+    "\n",
+    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
+    "from langchain.document_loaders.unstructured import UnstructuredFileLoader\n",
+    "from langchain.document_loaders.generic import GenericLoader\n",
+    "from langchain.document_loaders.parsers import OpenAIWhisperParser\n",
+    "from langchain.document_loaders.blob_loaders.youtube_audio import YoutubeAudioLoader\n",
+    "from langchain.document_loaders import WebBaseLoader, UnstructuredURLLoader\n",
+    "from langchain.docstore.document import Document\n",
+    "\n",
+    "from langchain.vectorstores import Chroma\n",
+    "from langchain.chains import RetrievalQAWithSourcesChain"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Note that we will not export the following packages to our module because in this exploration we have decided to go with langchain implementations, or they are only used for testing."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#exploration\n",
+    "import trafilatura\n",
+    "import requests\n",
+    "import justext"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Media to Text Converters\n",
+    "In this section, we provide a set of converters that can either read text and convert it to other useful text, or read YouTube or Websites and convert them into text."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Standard Text Splitter\n",
+    "Here we define a standard text splitter. This can be used on any text."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#| export\n",
+    "def rawtext_to_doc_split(text, chunk_size=1500, chunk_overlap=150):\n",
+    "  \n",
+    "  # Quick type checking\n",
+    "  if not isinstance(text, list):\n",
+    "    text = [text]\n",
+    "\n",
+    "  # Create splitter\n",
+    "  text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size,\n",
+    "                                                 chunk_overlap=chunk_overlap,\n",
+    "                                                 add_start_index = True)\n",
+    "  \n",
+    "  #Split into docs segments\n",
+    "  if isinstance(text[0], Document):\n",
+    "    doc_segments = text_splitter.split_documents(text)\n",
+    "  else:\n",
+    "    doc_segments = text_splitter.split_documents(text_splitter.create_documents(text))\n",
+    "\n",
+    "  # Make into one big list\n",
+    "  doc_segments = list(itertools.chain(*doc_segments)) if isinstance(doc_segments[0], list) else doc_segments\n",
+    "\n",
+    "  return doc_segments"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[Document(page_content='This is a', metadata={}),\n",
+       " Document(page_content='sentence.', metadata={}),\n",
+       " Document(page_content='This is', metadata={}),\n",
+       " Document(page_content='another', metadata={}),\n",
+       " Document(page_content='sentence.', metadata={}),\n",
+       " Document(page_content='This is a', metadata={}),\n",
+       " Document(page_content='a third', metadata={}),\n",
+       " Document(page_content='sentence.', metadata={})]"
+      ]
+     },
+     "execution_count": null,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# test basic functionality\n",
+    "rawtext_to_doc_split([\"This is a sentence. This is another sentence.\", \"This is a third sentence.\"], chunk_size=10, chunk_overlap=5)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "We'll write a quick function to do a unit test on the function we just wrote."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def test_split_texts():\n",
+    "    \n",
+    "    # basic behavior\n",
+    "    text = \"This is a sample text that we will use to test the splitter function.\"\n",
+    "    expected_output = [\"This is a sample text that we will use to test the splitter function.\"]\n",
+    "    out_splits = [doc.page_content for doc in rawtext_to_doc_split(text)]\n",
+    "    assert all([target==expected for target, expected in zip(expected_output, out_splits)]), ('The basic splitter functionality is incorrect, and does not correctly ' +\n",
+    "                                                                                              'use chunk_size and chunk_overlap on chunks <1500.')\n",
+    "    \n",
+    "    # try a known result with variable chunk_length and chunk_overlap\n",
+    "    text = (\"This is a sample text that we will use to test the splitter function. It should split the \" +\n",
+    "            \"text into multiple chunks of size 1500 with an overlap of 150 characters. This is the second chunk.\")\n",
+    "    expected_output = ['This is a sample text that we will use to test the',\n",
+    "                       'test the splitter function. It should split the',\n",
+    "                       'split the text into multiple chunks of size 1500',\n",
+    "                       'size 1500 with an overlap of 150 characters. This',\n",
+    "                       'This is the second chunk.']\n",
+    "    out_splits = [doc.page_content for doc in rawtext_to_doc_split(text, 50, 10)]\n",
+    "    assert all([target==expected for target, expected in zip(expected_output, out_splits)]), 'The splitter does not correctly use chunk_size and chunk_overlap.'\n",
+    "\n",
+    "# Run test\n",
+    "test_split_texts()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "The following function is used for testing to make sure single files and lists can be accommodated, and that what are returned are lists of documents."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# a set of tests to make sure that this works on both lists single inputs\n",
+    "def test_converters_inputs(test_fcn, files_list=None):\n",
+    "    if files_list is None:\n",
+    "        single_file = 'The cat was super cute and adorable'\n",
+    "        multiple_files = [single_file, 'The dog was also cute and her wet nose is always so cold!']\n",
+    "    elif isinstance(files_list, str):\n",
+    "        single_file = files_list\n",
+    "        multiple_files = [single_file, single_file]\n",
+    "    elif isinstance(files_list, list):\n",
+    "        single_file = files_list[0]\n",
+    "        multiple_files = files_list\n",
+    "    else:\n",
+    "        TypeError(\"You've passed in a files_list which is neither a string or a list or None\")\n",
+    "\n",
+    "    # test for single file\n",
+    "    res = test_fcn(single_file)\n",
+    "    assert isinstance(res, list), 'FAILED ASSERT in {test_fcn}. A single file should return a list.'\n",
+    "    assert not isinstance(res[0], list), 'FAILED ASSERT in {test_fcn}. A single file should return a 1-dimensional list.'\n",
+    "\n",
+    "    # test for multiple files\n",
+    "    res = test_fcn(multiple_files)\n",
+    "    assert isinstance(res, list), 'FAILED ASSERT in {test_fcn}. A list of files should return a list.'\n",
+    "    assert not isinstance(res[0], list), 'FAILED ASSERT in {test_fcn}. A list of files should return a 1-dimensional list with all documents combined.'\n",
+    "\n",
+    "    # test that the return type of elements should be Document\n",
+    "    assert all([isinstance(doc, Document) for doc in res]), 'FAILED ASSERT in {test_fcn}. The return type of elements should be Document.'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# test behavior of standard text splitter\n",
+    "test_converters_inputs(rawtext_to_doc_split)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### File or Files\n",
+    "Functions which load a single file or files from a directory, including pdfs, text files, html, images, and more. See [Unstructured File Documentation](https://python.langchain.com/docs/integrations/document_loaders/unstructured_file) for more information."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#| export\n",
+    "## A single File\n",
+    "def _file_to_text(single_file, chunk_size = 1000, chunk_overlap=150):\n",
+    "\n",
+    "  # Create loader and get segments\n",
+    "  loader = UnstructuredFileLoader(single_file)\n",
+    "  doc_segments = loader.load_and_split(RecursiveCharacterTextSplitter(chunk_size=chunk_size,\n",
+    "                                                                      chunk_overlap=chunk_overlap,\n",
+    "                                                                      add_start_index=True))\n",
+    "  return doc_segments\n",
+    "\n",
+    "\n",
+    "## Multiple files\n",
+    "def files_to_text(files_list, chunk_size=1000, chunk_overlap=150):\n",
+    "  \n",
+    "  # Quick type checking\n",
+    "  if not isinstance(files_list, list):\n",
+    "    files_list = [files_list]\n",
+    "\n",
+    "  # This is currently a fix because the UnstructuredFileLoader expects a list of files yet can't split them correctly yet\n",
+    "  all_segments = [_file_to_text(single_file, chunk_size=chunk_size, chunk_overlap=chunk_overlap) for single_file in files_list]\n",
+    "  all_segments = list(itertools.chain(*all_segments)) if isinstance(all_segments[0], list) else all_segments\n",
+    "\n",
+    "  return all_segments"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[Document(page_content='Two roads diverged in a yellow wood,\\rAnd sorry I could not travel both\\rAnd be one traveler, long I', metadata={'source': '../roadnottaken.txt', 'start_index': 0}),\n",
+       " Document(page_content='traveler, long I stood\\rAnd looked down one as far as I could\\rTo where it bent in the', metadata={'source': '../roadnottaken.txt', 'start_index': 82}),\n",
+       " Document(page_content='it bent in the undergrowth;\\r\\rThen took the other, as just as fair,\\rAnd having perhaps the better', metadata={'source': '../roadnottaken.txt', 'start_index': 152}),\n",
+       " Document(page_content='perhaps the better claim,\\rBecause it was grassy and wanted wear;\\rThough as for that the passing', metadata={'source': '../roadnottaken.txt', 'start_index': 230}),\n",
+       " Document(page_content='that the passing there\\rHad worn them really about the same,\\r\\rAnd both that morning equally lay\\rIn', metadata={'source': '../roadnottaken.txt', 'start_index': 309}),\n",
+       " Document(page_content='equally lay\\rIn leaves no step had trodden black. Oh, I kept the first for another day! Yet knowing', metadata={'source': '../roadnottaken.txt', 'start_index': 392}),\n",
+       " Document(page_content='day! Yet knowing how way leads on to way,\\rI doubted if I should ever come back. I shall be telling', metadata={'source': '../roadnottaken.txt', 'start_index': 474}),\n",
+       " Document(page_content='I shall be telling this with a sigh\\rSomewhere ages and ages hence:\\rTwo roads diverged in a wood,', metadata={'source': '../roadnottaken.txt', 'start_index': 554}),\n",
+       " Document(page_content='diverged in a wood, and IэI took the one less traveled by,\\rAnd that has made all the difference.', metadata={'source': '../roadnottaken.txt', 'start_index': 631}),\n",
+       " Document(page_content='Two roads diverged in a yellow wood,\\rAnd sorry I could not travel both\\rAnd be one traveler, long I', metadata={'source': '../roadnottaken.txt', 'start_index': 0}),\n",
+       " Document(page_content='traveler, long I stood\\rAnd looked down one as far as I could\\rTo where it bent in the', metadata={'source': '../roadnottaken.txt', 'start_index': 82})]"
+      ]
+     },
+     "execution_count": null,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# ensure basic behavior\n",
+    "res = files_to_text(['../roadnottaken.txt', '../roadnottaken.txt'], chunk_size=100, chunk_overlap=20)\n",
+    "res[:11]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "test_converters_inputs(files_to_text, '../roadnottaken.txt')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Youtube\n",
+    "This works by first transcribing the video to text."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#| export\n",
+    "def youtube_to_text(urls, save_dir = \"content\"):\n",
+    "  # Transcribe the videos to text\n",
+    "  # save_dir: directory to save audio files\n",
+    "\n",
+    "  if not isinstance(urls, list):\n",
+    "    urls = [urls]\n",
+    "  \n",
+    "  youtube_loader = GenericLoader(YoutubeAudioLoader(urls, save_dir), OpenAIWhisperParser())\n",
+    "  youtube_docs = youtube_loader.load()\n",
+    "  \n",
+    "  return youtube_docs"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Now, let's demonstrate functionality using some existing YouTube videos"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Two Karpathy lecture videos\n",
+    "urls = [\"https://youtu.be/kCc8FmEb1nY\", \"https://youtu.be/VMj-3S1tku0\"]\n",
+    "youtube_text = youtube_to_text(urls)\n",
+    "youtube_text"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Other Youtube helper functions to help with getting full features of YouTube videos are included below. These two grab and save the text of the transcripts.\n",
+    "\n",
+    "<p style=\"color:red\"><strong>Note that in this stage of development, the following cannot be tested due to YouTube download errors.</strong></p>"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#| export\n",
+    "def save_text(text, text_name = None):\n",
+    "  if not text_name:\n",
+    "    text_name = text[:20]\n",
+    "  text_path = os.path.join(\"/content\",text_name+\".txt\")\n",
+    "  \n",
+    "  with open(text_path, \"x\") as f:\n",
+    "    f.write(text)\n",
+    "  # Return the location at which the transcript is saved\n",
+    "  return text_path"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#| export\n",
+    "def get_youtube_transcript(yt_url, save_transcript = False, temp_audio_dir = \"sample_data\"):\n",
+    "  # Transcribe the videos to text and save to file in /content\n",
+    "  # save_dir: directory to save audio files\n",
+    "\n",
+    "  youtube_docs = youtube_to_text(yt_url, save_dir = temp_audio_dir)\n",
+    "  \n",
+    "  # Combine doc\n",
+    "  combined_docs = [doc.page_content for doc in youtube_docs]\n",
+    "  combined_text = \" \".join(combined_docs)\n",
+    "  \n",
+    "  # Save text to file\n",
+    "  video_path = youtube_docs[0].metadata[\"source\"]\n",
+    "  youtube_name = os.path.splitext(os.path.basename(video_path))[0]\n",
+    "\n",
+    "  save_path = None\n",
+    "  if save_transcript:\n",
+    "    save_path = save_text(combined_text, youtube_name)\n",
+    "  \n",
+    "  return youtube_docs, save_path"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Websites\n",
+    "We have a few different approaches to reading website text. Some approaches are specifically provided through langchain and some are other packages that seem to be performant. We'll show the pros/cons of each approach below.\n",
+    "\n",
+    "#### Langchain: WebBaseLoader"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#| export\n",
+    "def website_to_text_web(url, chunk_size = 1500, chunk_overlap=100):\n",
+    "  \n",
+    "    # Url can be a single string or list\n",
+    "    website_loader = WebBaseLoader(url)\n",
+    "    website_raw = website_loader.load()\n",
+    "\n",
+    "    website_data = rawtext_to_doc_split(website_raw, chunk_size = chunk_size, chunk_overlap=chunk_overlap)\n",
+    "  \n",
+    "    # Combine doc\n",
+    "    return website_data"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Now for a quick test to ensure functionality..."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "demo_urls = [\"https://www.espn.com/\", \"https://www.vanderbilt.edu/undergrad-datascience/faq\"]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[Document(page_content=\"ESPN - Serving Sports Fans. Anytime. Anywhere.\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n        Skip to main content\\n    \\n\\n        Skip to navigation\\n    \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n<\\n\\n>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nMenuESPN\\n\\n\\nSearch\\n\\n\\n\\nscores\\n\\n\\n\\nNFLMLBNBANHLSoccerGolf…Women's World CupNCAAFNCAAMNCAAWSports BettingBoxingCFLNCAACricketF1HorseMMANASCARNBA G LeagueOlympic SportsPLLRacingRN BBRN FBRugbyTennisWNBAWWEX GamesXFLMore ESPNFantasyListenWatchESPN+\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n  \\n\\nSUBSCRIBE NOW\\n\\n\\n\\n\\n\\nPaul vs. Diaz (ESPN+ PPV)\\n\\n\\n\\n\\n\\n\\n\\nPGA TOUR LIVE\\n\\n\\n\\n\\n\\n\\n\\nLittle League Baseball: Regionals\\n\\n\\n\\n\\n\\n\\n\\nMLB: Select Games\\n\\n\\n\\n\\n\\n\\n\\nCrossFit Games\\n\\n\\n\\n\\n\\n\\n\\nSlamBall\\n\\n\\n\\n\\n\\n\\n\\nThe Ultimate Fighter: Season 31\\n\\n\\n\\n\\n\\n\\n\\nFantasy Football: Top Storylines, Rookies, Sleepers\\n\\n\\nQuick Links\\n\\n\\n\\n\\nWomen's World Cup\\n\\n\\n\\n\\n\\n\\n\\nNHL Free Agency\\n\\n\\n\\n\\n\\n\\n\\nNBA Free Agency Buzz\\n\\n\\n\\n\\n\\n\\n\\nNBA Trade Machine\\n\\n\\n\\n\\n\\n\\n\\nThe Basketball Tournament\\n\\n\\n\\n\\n\\n\\n\\nFantasy Football: Sign Up\\n\\n\\n\\n\\n\\n\\n\\nHow To Watch PGA TOUR\\n\\n\\n\\n\\n\\n\\nFavorites\\n\\n\\n\\n\\n\\n\\n      Manage Favorites\\n      \\n\\n\\n\\nCustomize ESPNSign UpLog InESPN Sites\\n\\n\\n\\n\\nESPN Deportes\\n\\n\\n\\n\\n\\n\\n\\nAndscape\\n\\n\\n\\n\\n\\n\\n\\nespnW\\n\\n\\n\\n\\n\\n\\n\\nESPNFC\\n\\n\\n\\n\\n\\n\\n\\nX Games\\n\\n\\n\\n\\n\\n\\n\\nSEC Network\\n\\n\\nESPN Apps\\n\\n\\n\\n\\nESPN\\n\\n\\n\\n\\n\\n\\n\\nESPN Fantasy\\n\\n\\nFollow ESPN\\n\\n\\n\\n\\nFacebook\\n\\n\\n\\n\\n\\n\\n\\nX/Twitter\\n\\n\\n\\n\\n\\n\\n\\nInstagram\\n\\n\\n\\n\\n\\n\\n\\nSnapchat\\n\\n\\n\\n\\n\\n\\n\\nTikTok\\n\\n\\n\\n\\n\\n\\n\\nYouTube\", metadata={'source': 'https://www.espn.com/', 'title': 'ESPN - Serving Sports Fans. Anytime. Anywhere.', 'description': 'Visit ESPN for live scores, highlights and sports news. Stream exclusive games on ESPN+ and play fantasy sports.', 'language': 'en'}),\n",
+       " Document(page_content=\"How can your team win the national title? Connelly breaks down what needs to go right for all 17 contendersThe fewer things that have to go right to win a title, the better a team's chances of taking the crown. Here's what has to fall each contender's way.7hBill ConnellyDale Zanine/USA TODAY SportsPosition U 2023: Is USC on the verge of taking over QBU from Oklahoma?Which schools produce the most talent at each position?1dDavid HaleConnelly's conference previews: Intel on all 133 FBS teamsTOP HEADLINESFreeze 'uncomfortable' as Auburn opens campTexans' Metchie relied on faith amid cancer fightHornets have new owners after MJ sale finalizedMiami coach expects rough treatment of MessiDrexel basketball player found dead in apartmentGermany exits WWC after draw with South KoreaBrady takes minority stake in English soccer teamDeep dish: Cubs' output at plate best since 1897Re-drafting 2018 NFL class 5 years laterWHAT HAPPENED IN INDY?Inside the shocking feud between Jonathan Taylor and the ColtsHe was the NFL's leading rusher two seasons ago and wanted an extension with the Colts, but now he wants out. How things got so bad for Taylor and Indianapolis.8hStephen HolderZach Bolinger/Icon Sportswire'THE BEST IN THE WORLD RIGHT NOW'Why Stephen A. is convinced Tyreek Hill is the NFL's top WR2h2:57WYNDHAM CHAMPIONSHIPCONTINUES THROUGH SUNDAYShane Lowry fluffs shot, drains birdie chip immediately after3h0:35Countdown to FedEx Cup Playoffs, AIG Open and the Ryder CupDiana Taurasi, 10,000\", metadata={'source': 'https://www.espn.com/', 'title': 'ESPN - Serving Sports Fans. Anytime. Anywhere.', 'description': 'Visit ESPN for live scores, highlights and sports news. Stream exclusive games on ESPN+ and play fantasy sports.', 'language': 'en'}),\n",
+       " Document(page_content=\"Lowry fluffs shot, drains birdie chip immediately after3h0:35Countdown to FedEx Cup Playoffs, AIG Open and the Ryder CupDiana Taurasi, 10,000 points and the shot that made WNBA scoring history1dMLB SCOREBOARDTHURSDAY'S GAMESSee AllTrivia: Can you guess the right player?HERE COMES HELPBring on the reinforcements! 10 returning players as good as a trade deadline blockbusterInjured stars expected to come off the IL soon -- or have already -- could rock MLB's playoff races.7hAlden GonzalezJay Biggerstaff-USA TODAY Sports'CLEARLY THE ACC IS STRUGGLING'Finebaum: FSU is better off leaving the ACC5h1:04Thamel's realignment buzz: Latest on Pac-12, Big 12 and ACCAN AGGRESSIVE STRATEGYHow the Big 12 landed Colorado and shook up college footballThe Big 12 learned lessons two years ago after getting burned by Texas and Oklahoma. It resulted in a more aggressive strategy that could dramatically change the sport.2dHeather DinichRaymond Carlin/Icon Sportswire Top HeadlinesFreeze 'uncomfortable' as Auburn opens campTexans' Metchie relied on faith amid cancer fightHornets have new owners after MJ sale finalizedMiami coach expects rough treatment of MessiDrexel basketball player found dead in apartmentGermany exits WWC after draw with South KoreaBrady takes minority stake in English soccer teamDeep dish: Cubs' output at plate best since 1897Re-drafting 2018 NFL class 5 years laterFavorites FantasyManage FavoritesFantasy HomeCustomize ESPNSign UpLog InICYMI0:54Serena Williams, Alexis Ohanian\", metadata={'source': 'https://www.espn.com/', 'title': 'ESPN - Serving Sports Fans. Anytime. Anywhere.', 'description': 'Visit ESPN for live scores, highlights and sports news. Stream exclusive games on ESPN+ and play fantasy sports.', 'language': 'en'}),\n",
+       " Document(page_content='2018 NFL class 5 years laterFavorites FantasyManage FavoritesFantasy HomeCustomize ESPNSign UpLog InICYMI0:54Serena Williams, Alexis Ohanian use drones to reveal gender of 2nd childSerena Williams and her husband Alexis Ohanian find out the gender of their second child in a spectacular display of drones. Best of ESPN+Todd Kirkland/Getty ImagesMLB 2023 trade deadline: Winners, losers and in-betweenersThe 2023 trade deadline is over! Who crushed it, and who left much to be desired? We weigh in on all 30 clubs.AP Photo/Matt YorkLowe: Why Bradley Beal could unlock KD, Book and the most dangerous version of the Phoenix Suns yetWith Kevin Durant, Devin Booker and Beal, Phoenix is already an inner-circle title contender. But if the Suns continue a Beal experiment the Wizards ran last season? Good luck.Cliff Welch/Icon SportswirePredicting 10 NFL starting quarterback battles: Who is QB1?We talked to people around the NFL and projected the QB1 for 10 unsettled situations, including a wide-open race in Tampa Bay. Trending NowAP Photo/Julio Cortez\\'Revis Island\\' resonates long after Hall of Famer\\'s retirementDarrelle Revis made his name as a dominant corner but might be best known for his \"island\" moniker players still adopt today.Illustration by ESPNThe wild life of Gardner MinshewFour colleges, three NFL teams, two Manias and the hug that broke the internet. It\\'s been an unbelievable ride for Gardner Minshew. Next stop: Indianapolis.Illustration by ESPNBest 2023 Women\\'s World Cup', metadata={'source': 'https://www.espn.com/', 'title': 'ESPN - Serving Sports Fans. Anytime. Anywhere.', 'description': 'Visit ESPN for live scores, highlights and sports news. Stream exclusive games on ESPN+ and play fantasy sports.', 'language': 'en'}),\n",
+       " Document(page_content=\"that broke the internet. It's been an unbelievable ride for Gardner Minshew. Next stop: Indianapolis.Illustration by ESPNBest 2023 Women's World Cup players: Morgan, Caicedo, moreESPN's expert panel selected the top 25 players of the Women's World Cup to keep an eye on, from Sophia Smith to Sam Kerr and more. How to Watch on ESPN+(AP Photo/Koji Sasahara, File)How to watch the PGA Tour, Masters, PGA Championship and FedEx Cup playoffs on ESPN, ESPN+Here's everything you need to know about how to watch the PGA Tour, Masters, PGA Championship and FedEx Cup playoffs on ESPN and ESPN+. Sign up to play the #1 Fantasy game!Create A LeagueJoin Public LeagueReactivateMock Draft NowSign up for FREE!Create A LeagueJoin a Public LeagueReactivate a LeaguePractice With a Mock DraftSign up for FREE!Create A LeagueJoin a Public LeagueReactivate a LeaguePractice with a Mock Draft\", metadata={'source': 'https://www.espn.com/', 'title': 'ESPN - Serving Sports Fans. Anytime. Anywhere.', 'description': 'Visit ESPN for live scores, highlights and sports news. Stream exclusive games on ESPN+ and play fantasy sports.', 'language': 'en'}),\n",
+       " Document(page_content=\"ESPN+\\n\\n\\n\\n\\nPaul vs. Diaz (ESPN+ PPV)\\n\\n\\n\\n\\n\\n\\n\\nPGA TOUR LIVE\\n\\n\\n\\n\\n\\n\\n\\nLittle League Baseball: Regionals\\n\\n\\n\\n\\n\\n\\n\\nMLB: Select Games\\n\\n\\n\\n\\n\\n\\n\\nCrossFit Games\\n\\n\\n\\n\\n\\n\\n\\nSlamBall\\n\\n\\n\\n\\n\\n\\n\\nThe Ultimate Fighter: Season 31\\n\\n\\n\\n\\n\\n\\n\\nFantasy Football: Top Storylines, Rookies, Sleepers\\n\\n\\nQuick Links\\n\\n\\n\\n\\nWomen's World Cup\\n\\n\\n\\n\\n\\n\\n\\nNHL Free Agency\\n\\n\\n\\n\\n\\n\\n\\nNBA Free Agency Buzz\\n\\n\\n\\n\\n\\n\\n\\nNBA Trade Machine\\n\\n\\n\\n\\n\\n\\n\\nThe Basketball Tournament\\n\\n\\n\\n\\n\\n\\n\\nFantasy Football: Sign Up\\n\\n\\n\\n\\n\\n\\n\\nHow To Watch PGA TOUR\\n\\n\\nESPN Sites\\n\\n\\n\\n\\nESPN Deportes\\n\\n\\n\\n\\n\\n\\n\\nAndscape\\n\\n\\n\\n\\n\\n\\n\\nespnW\\n\\n\\n\\n\\n\\n\\n\\nESPNFC\\n\\n\\n\\n\\n\\n\\n\\nX Games\\n\\n\\n\\n\\n\\n\\n\\nSEC Network\\n\\n\\nESPN Apps\\n\\n\\n\\n\\nESPN\\n\\n\\n\\n\\n\\n\\n\\nESPN Fantasy\\n\\n\\nFollow ESPN\\n\\n\\n\\n\\nFacebook\\n\\n\\n\\n\\n\\n\\n\\nX/Twitter\\n\\n\\n\\n\\n\\n\\n\\nInstagram\\n\\n\\n\\n\\n\\n\\n\\nSnapchat\\n\\n\\n\\n\\n\\n\\n\\nTikTok\\n\\n\\n\\n\\n\\n\\n\\nYouTube\\n\\n\\nTerms of UsePrivacy PolicyYour US State Privacy RightsChildren's Online Privacy PolicyInterest-Based AdsAbout Nielsen MeasurementDo Not Sell or Share My Personal InformationContact UsDisney Ad Sales SiteWork for ESPNCopyright: © ESPN Enterprises, Inc. All rights reserved.\", metadata={'source': 'https://www.espn.com/', 'title': 'ESPN - Serving Sports Fans. Anytime. Anywhere.', 'description': 'Visit ESPN for live scores, highlights and sports news. Stream exclusive games on ESPN+ and play fantasy sports.', 'language': 'en'}),\n",
+       " Document(page_content='Frequently Asked Questions |   Undergraduate Data Science | Vanderbilt University\\n\\n\\n\\n\\n \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nSkip to main content\\n\\nlink\\n\\n\\n\\n\\n\\nHome\\nPeople\\nMinor\\n\\nMinor Requirements\\nCourse Descriptions\\nCourse Schedule\\nHow to Declare the Minor\\nChoosing a Minor\\n\\n\\nResearch and Immersion\\n\\nResearch and Immersion Overview\\nDS 3850 Research in Data Science\\nDSI Summer Research Program\\nData Science for Social Good\\nResearch Immersion in Data Science\\nDSI Internship\\n\\n\\nFAQ\\nNews\\nForms\\nContact and Email List\\nData Science Institute\\n \\n\\n\\n\\n\\n\\n\\n\\t\\t\\t\\t\\t\\tUndergraduate Data Science \\n\\n\\n\\n\\n\\n\\n\\n\\n\\nFrequently Asked Questions\\nDeclaring the Minor\\n\\n\\n\\nHow do I declare the Data Science Minor?Use the forms and follow the procedures for your home college. See How to Declare the Data Science Minor.\\n\\n\\nWhen should I declare the Data Science Minor?While minor declarations can be made any time, DS courses will give some preference to students who have officially declared the Data Science Minor. So we recommend declaring the minor sooner rather than later. It is always possible to drop a declared minor. Minor declarations must be submitted at least two weeks before registration begins. Otherwise, the minor declaration will not be processed until after registration. No preference will be given during registration for an “intent” to declare because the minor declaration was made too late.', metadata={'source': 'https://www.vanderbilt.edu/undergrad-datascience/faq', 'title': 'Frequently Asked Questions |   Undergraduate Data Science | Vanderbilt University', 'description': 'Frequently Asked Questions. ', 'language': 'en'}),\n",
+       " Document(page_content='I declared the Data Science Minor, but I did not get into the class I wanted to take for the minor. Why?First, preference for students who have declared the minor only applies to DS courses, not other courses. Second, if you declared the minor within two weeks of registration, your minor declaration will. not show up on YES, and you will not have preference. Third, while we try to hold as many seats for students who have declared the minor as we can, not all seats are reserved.\\n\\n\\nI am a first-year A&S student. Can I really declare the Data Science Minor now?Yes. While A&S students are usually prevented from declaring a major or minor until sophomore year, first-year A&S students can declare the Data Science Minor. As noted in the previous question, this can be important to do since some popular core DS courses will give some preference to students who have officially declared Data Science as a minor.', metadata={'source': 'https://www.vanderbilt.edu/undergrad-datascience/faq', 'title': 'Frequently Asked Questions |   Undergraduate Data Science | Vanderbilt University', 'description': 'Frequently Asked Questions. ', 'language': 'en'}),\n",
+       " Document(page_content='I am a current junior (rising senior), can I complete the Data Science Minor (for Spring 2021 juniors only)?Juniors must contact the Director of Undergraduate Data Science to discuss options. DS 1000 is not open to current juniors (rising seniors). DS 3100 will not be taught next year (Fall 2021 or Spring 2022) and will need to be suitably replaced, which will require an approved plan from the Director. Furthermore, while DS / CS 3262 is current slated to be taught Spring 2022, that is not fully guaranteed, so students should see if they can take one of the other machine learning options.\\n\\n\\nI am a rising senior or current senior and cannot register for DS 1000. Why?Rising seniors and current seniors can only register for DS 1000 if there are available seats immediately before the semester begins with permission of the instructor. DS 1000 is intended as an introduction to data science for first years and sophomores, which is why this restriction is in place.\\n\\n\\nCollege-Specific Information\\n\\n\\n\\nWhat college is the home of the Data Science Minor?The Data Science Minor is a trans-institutional minor, shared by A&S, Blair, Engineering, and Peabody.', metadata={'source': 'https://www.vanderbilt.edu/undergrad-datascience/faq', 'title': 'Frequently Asked Questions |   Undergraduate Data Science | Vanderbilt University', 'description': 'Frequently Asked Questions. ', 'language': 'en'}),\n",
+       " Document(page_content='I am an A&S student. Do DS courses count as A&S courses?All courses with a DS prefix count as courses within each of the colleges, including A&S. If you are an A&S student, and are taking a course that is cross-listed, make sure you enroll in the one with the DS prefix. Electives outside of A&S without the DS prefix will generally not count as A&S courses, so plan accordingly.\\n\\n\\nWhat are the unique credit hour rules for the Data Science Minor?Students electing an undergraduate minor in Data Science must follow academic regulations regarding minors in their home college, including but not limited to regulations regarding unique hours. The unique credit hour rule is specific to the College of Arts and Science and Peabody College. The School of Engineering and Blair School of Music do not have a unique credit hour rule. The Data Science minor cannot waive this rule. Please talk with your academic advisor about how to satisfy these requirements.\\n\\n\\nInfo About the Courses\\n\\n\\n\\nDS 1000Thank you for your interest in DS 1000! The course is full for the fall 2021 semester. Due to student demand and the transinstitutional nature of the course, we cannot make special exceptions as to which students, if any, on the waitlist are able to enroll. DS 1000 will be offered again in the spring semester.', metadata={'source': 'https://www.vanderbilt.edu/undergrad-datascience/faq', 'title': 'Frequently Asked Questions |   Undergraduate Data Science | Vanderbilt University', 'description': 'Frequently Asked Questions. ', 'language': 'en'}),\n",
+       " Document(page_content='What computer programming course should I take?See What Programming Course To Take? In general, students interested in data science and scientific computing (not in computer science per se) should learn Python (and R).\\n\\n\\nHow do I find courses approved for the data science minor on YES?On YES, to select all courses approved for credit in the Data Science minor offered in a given semester, select the “Advanced” link next to the search box, select the “Class Attributes” drop-down box on the bottom right of the advanced search page, and then select “Eligible for Data Science” to find all courses. (Note that these course tags will not all be in place on YES until the registration period for Fall 2021 begins.)\\n\\n\\nCan other courses, besides those listed, count towards the Data Science Minor?New courses, special topics courses, or graduate-level courses that seem related to data science could count as electives. Contact the Director of Undergraduate Data Science to request consideration.', metadata={'source': 'https://www.vanderbilt.edu/undergrad-datascience/faq', 'title': 'Frequently Asked Questions |   Undergraduate Data Science | Vanderbilt University', 'description': 'Frequently Asked Questions. ', 'language': 'en'}),\n",
+       " Document(page_content='Why doesn’t CS 1104 count towards the Data Science Minor?It does, as a prerequisite to CS 2204, which counts towards the minor. CS / DS 1100 was created as a new single-semester programming course for the Data Science Minor. It roughly has 2/3 the content of CS 1104 and 1/3 the content of CS 2204. While CS / DS 1100 counts as a single semester of programming for the minor, we strongly encourage students interested in data science, and in using data science tools and techniques, to take two semesters of programming in Python (CS / DS 1100 or CS 1104, followed by CS 2204). If you have taken CS 1104, you can take CS 1100, but you will only receive a total of four credits for\\xa0the two courses. See also What Programming Course To Take?', metadata={'source': 'https://www.vanderbilt.edu/undergrad-datascience/faq', 'title': 'Frequently Asked Questions |   Undergraduate Data Science | Vanderbilt University', 'description': 'Frequently Asked Questions. ', 'language': 'en'}),\n",
+       " Document(page_content='I see that after having taken CS 1104, I can take CS/DS 1100 instead of taking CS 2204. What are the downsides of doing so?After taking CS 1104, we do recommend you take CS 2204. If you are interested in data science, a broader experience in Python in desirable (in fact, we recommend that students having taken CS 1100 try to take CS 2204 as well). CS/DS 1100 and 1104 have significant overlap (both are introductions to programming using Python). That said, it is permissible to take CS/DS 1100 after having taken CS 1104. You will only get 1 (out of 3) credit hours for CS/DS 1100 (after having taken CS 1104), but the combination of CS/DS 1100 and 1104 will satisfy the DS minor programming requirement. Note that if you enroll in three 3-hour courses and CS/DS 1100 (after having taken CS 1104) it will look like you are registered for 12 credit hours during registration and at the start of the semester, but your credit hours will be reduced to only 10 credit hours (because the credits for CS/DS 1100 will be cut back to 1 after the add/drop period). Enrolling in fewer than 12 credit hours can have significant consequences on financial aid and potentially on visa status for international students. Please be mindful of this.\\n\\n\\nWhat is the difference between CS 1100 and DS 1100?Nothing. They are the same course. They meet the same time in the same place and are taught by the same instructor. They are just cross-listed.', metadata={'source': 'https://www.vanderbilt.edu/undergrad-datascience/faq', 'title': 'Frequently Asked Questions |   Undergraduate Data Science | Vanderbilt University', 'description': 'Frequently Asked Questions. ', 'language': 'en'}),\n",
+       " Document(page_content='I have taken CS 1101. What computer programming course should I take next?You have two options. You can either take CS 2201 (in C++) or take CS 1100 (in Python). Of course, you could also take CS 1104 and 2204 (in Python). CS 1100, 2201, and 2204 all satisfy the programming requirement for the minor. Note that CS 2201 is a prerequisite for many upper-level CS courses (as well as required for the CS major and minor). For more information, see What Programming Course To Take?\\n\\n\\nECON 3750 and MATH 3670 are listed both as satisfying the core machine learning requirement and as electives. If I take one, will it double-count for both requirements?No. They are listed under both because a student who takes one of the other machine learning\\xa0courses to satisfy the core requirement (CS/DS 3262 or CS 4262) can also take ECON 3750 or MATH 3670 as an elective; the content is sufficiently different that both can count towards the minor, but one course cannot double-count for two minor requirements.', metadata={'source': 'https://www.vanderbilt.edu/undergrad-datascience/faq', 'title': 'Frequently Asked Questions |   Undergraduate Data Science | Vanderbilt University', 'description': 'Frequently Asked Questions. ', 'language': 'en'}),\n",
+       " Document(page_content='Can I take ECON 3750 or MATH 3670 as an elective if I have already taken CS 3262 or CS 4262?Yes (see above). ECON 3750 and MATH 3670 are sufficiently different from CS 3262 or CS 4262 (and from each other) that you can take these as electives. In fact, you could take ECON 3750 to satisfy the machine learning requirement and then take MATH 3670 as an elective.\\nCS 3262 can count towards the Data Science minor. CS 3262 does not count directly towards the Computer Science major requirements but could be used as either a tech elective or open elective for Computer Science majors.\\n\\n\\nWhy doesn’t MATH 2820 count towards the Data Science Minor?It does, as a prerequisite to MATH 2821, which counts towards the minor. The two-course sequence of MATH 2820 and MATH 2821 counts towards the Data Science Minor; the\\xa0two-course sequence is required because MATH 2820 goes deep into mathematical foundations of probability ad statistics concepts, but does not by itself cover the breadth of topics of other introductory statistics courses. This two-course sequence provides an excellent introduction to mathematical statistics.\\n\\n\\nResearch and Immersion Information\\n\\n\\n\\nCan I do research for course credit?Yes, you can do research for course credit (including DS 3850). More information can be found here: https://www.vanderbilt.edu/undergrad-datascience/ds-3850-research-in-data-science/', metadata={'source': 'https://www.vanderbilt.edu/undergrad-datascience/faq', 'title': 'Frequently Asked Questions |   Undergraduate Data Science | Vanderbilt University', 'description': 'Frequently Asked Questions. ', 'language': 'en'}),\n",
+       " Document(page_content='I am interested in the Undergraduate Data Science Immersion Program. How can I participate.Some competitive summer immersion programs include DSI-SPR and Data Science for Social Good (DSSG). More information can be found on the following websites.\\n\\nhttps://www.vanderbilt.edu/datascience/academics/undergraduate/summer-research-program/\\nhttps://www.vanderbilt.edu/datascience/data-science-for-social-good/\\n\\nTo get involved in data-science-oriented research with a faculty member, you will need to reach out to the faculty member. Pointers can be found here: https://www.vanderbilt.edu/undergrad-datascience/research-and-immersion-overview/. Having that research count towards the immersion requirement will be between your faculty mentor and your faculty immersion coordinator.\\nAdditional information about research opportunities will be posted on the website in the future.\\n\\xa0\\n\\n\\nContact\\n\\n\\n\\nHow do I ask a question about the Data Science?If you have questions about the Data Science Minor or Immersion opportunities in data science, please email us: [email protected]\\n\\n\\nTo whom can I petition if the Director denies my request?The Governing Board of the Data Science Minor acts as the college-level oversight body for this trans-institutional minor and would be the appropriate next step for petitions related to the minor.\\n\\n\\n\\n\\n\\n\\n\\nData Science News\\n\\n\\n\\n Opportunities for Capstone Projects and Research Experience\\n\\n\\n\\n Attention Graduate Students! We’re Hiring!', metadata={'source': 'https://www.vanderbilt.edu/undergrad-datascience/faq', 'title': 'Frequently Asked Questions |   Undergraduate Data Science | Vanderbilt University', 'description': 'Frequently Asked Questions. ', 'language': 'en'}),\n",
+       " Document(page_content='Data Science News\\n\\n\\n\\n Opportunities for Capstone Projects and Research Experience\\n\\n\\n\\n Attention Graduate Students! We’re Hiring!\\n\\n\\n\\n Vanderbilt student-athlete drives sports performance through data analysis\\n\\n\\n\\n New Course: DS 3891 Special Topics: Intro to Generative AI\\n\\n\\n\\n Now Accepting Applications: DS Minor Teaching Fellowship for graduate students\\n\\n\\n\\n Join Our Team: Student Worker Positions Available for Fall 2023 Semester!\\n\\n\\n\\n\\n\\nVIEW MORE EVENTS >\\n\\n\\n\\n\\nYour Vanderbilt\\n\\nAlumni\\nCurrent Students\\nFaculty & Staff\\nInternational Students\\nMedia\\nParents & Family\\nProspective Students\\nResearchers\\nSports Fans\\nVisitors & Neighbors\\n\\n\\n\\n\\n \\n\\n\\n\\nQuick Links\\n\\nPeopleFinder\\nLibraries\\nNews\\nCalendar\\nMaps\\nA-Z\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n©\\n Vanderbilt University · All rights reserved. Site Development: Digital Strategies (Division of Communications)\\nVanderbilt University is committed to principles of equal opportunity and affirmative action. Accessibility information. Vanderbilt®, Vanderbilt University®, V Oak Leaf Design®, Star V Design® and Anchor Down® are trademarks of The Vanderbilt University', metadata={'source': 'https://www.vanderbilt.edu/undergrad-datascience/faq', 'title': 'Frequently Asked Questions |   Undergraduate Data Science | Vanderbilt University', 'description': 'Frequently Asked Questions. ', 'language': 'en'})]"
+      ]
+     },
+     "execution_count": null,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# get the results\n",
+    "res_web = website_to_text_web(demo_urls)\n",
+    "\n",
+    "res_web"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#unit testbed\n",
+    "test_converters_inputs(website_to_text_web, demo_urls)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Something interesting that we notice here is the proliferation of new lines that aren't for the best.\n",
+    "\n",
+    "#### Langchain: UnstructuredURLLoader"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#| export\n",
+    "def website_to_text_unstructured(web_urls, chunk_size = 1500, chunk_overlap=100):\n",
+    "\n",
+    "    # Make sure it's a list\n",
+    "    if not isinstance(web_urls, list):\n",
+    "        web_urls = [web_urls]\n",
+    "  \n",
+    "    # Url can be a single string or list\n",
+    "    website_loader = UnstructuredURLLoader(web_urls)\n",
+    "    website_raw = website_loader.load()\n",
+    "\n",
+    "    website_data = rawtext_to_doc_split(website_raw, chunk_size = chunk_size, chunk_overlap=chunk_overlap)\n",
+    "  \n",
+    "    # Return individual docs or list\n",
+    "    return website_data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[Document(page_content=\"Menu\\n\\nESPN\\n\\nSearch\\n\\n\\n\\nscores\\n\\nNFL\\n\\nMLB\\n\\nNBA\\n\\nNHL\\n\\nSoccer\\n\\nGolf\\n\\n…Women's World CupNCAAFNCAAMNCAAWSports BettingBoxingCFLNCAACricketF1HorseMMANASCARNBA G LeagueOlympic SportsPLLRacingRN BBRN FBRugbyTennisWNBAWWEX GamesXFL\\n\\nMore ESPN\\n\\nFantasy\\n\\nListen\\n\\nWatch\\n\\nESPN+\\n\\nSUBSCRIBE NOW\\n\\nPaul vs. Diaz (ESPN+ PPV)\\n\\nPGA TOUR LIVE\\n\\nLittle League Baseball: Regionals\\n\\nMLB: Select Games\\n\\nCrossFit Games\\n\\nSlamBall\\n\\nThe Ultimate Fighter: Season 31\\n\\nFantasy Football: Top Storylines, Rookies, Sleepers\\n\\nQuick Links\\n\\nWomen's World Cup\\n\\nNHL Free Agency\\n\\nNBA Free Agency Buzz\\n\\nNBA Trade Machine\\n\\nThe Basketball Tournament\\n\\nFantasy Football: Sign Up\\n\\nHow To Watch PGA TOUR\\n\\nFavorites\\n\\nManage Favorites\\n\\nCustomize ESPN\\n\\nESPN Sites\\n\\nESPN Deportes\\n\\nAndscape\\n\\nespnW\\n\\nESPNFC\\n\\nX Games\\n\\nSEC Network\\n\\nESPN Apps\\n\\nESPN\\n\\nESPN Fantasy\\n\\nFollow ESPN\\n\\nFacebook\\n\\nX/Twitter\\n\\nInstagram\\n\\nSnapchat\\n\\nTikTok\\n\\nYouTube\\n\\nHow can your team win the national title? Connelly breaks down what needs to go right for all 17 contendersThe fewer things that have to go right to win a title, the better a team's chances of taking the crown. Here's what has to fall each contender's way.7hBill ConnellyDale Zanine/USA TODAY Sports\\n\\nPosition U 2023: Is USC on the verge of taking over QBU from Oklahoma?Which schools produce the most talent at each position?1dDavid Hale\\n\\nConnelly's conference previews: Intel on all 133 FBS teams\\n\\nTOP HEADLINES\\n\\nFreeze 'uncomfortable' as Auburn opens camp\\n\\nTexans' Metchie relied on faith amid cancer fight\", metadata={'source': 'https://www.espn.com/'}),\n",
+       " Document(page_content=\"TOP HEADLINES\\n\\nFreeze 'uncomfortable' as Auburn opens camp\\n\\nTexans' Metchie relied on faith amid cancer fight\\n\\nHornets have new owners after MJ sale finalized\\n\\nMiami coach expects rough treatment of Messi\\n\\nDrexel basketball player found dead in apartment\\n\\nGermany exits WWC after draw with South Korea\\n\\nBrady takes minority stake in English soccer team\\n\\nDeep dish: Cubs' output at plate best since 1897\\n\\nRe-drafting 2018 NFL class 5 years later\\n\\nWHAT HAPPENED IN INDY?\\n\\nInside the shocking feud between Jonathan Taylor and the ColtsHe was the NFL's leading rusher two seasons ago and wanted an extension with the Colts, but now he wants out. How things got so bad for Taylor and Indianapolis.8hStephen HolderZach Bolinger/Icon Sportswire\\n\\n'THE BEST IN THE WORLD RIGHT NOW'\\n\\nWhy Stephen A. is convinced Tyreek Hill is the NFL's top WR\\n\\n2h\\n\\n2:57\\n\\nWYNDHAM CHAMPIONSHIP\\n\\nCONTINUES THROUGH SUNDAY\\n\\nShane Lowry fluffs shot, drains birdie chip immediately after\\n\\n4h\\n\\n0:35\\n\\nCountdown to FedEx Cup Playoffs, AIG Open and the Ryder Cup\\n\\nDiana Taurasi, 10,000 points and the shot that made WNBA scoring history1d\\n\\nMLB SCOREBOARDTHURSDAY'S GAMES\\n\\nSee All\\n\\nTrivia: Can you guess the right player?\\n\\nHERE COMES HELP\\n\\nBring on the reinforcements! 10 returning players as good as a trade deadline blockbusterInjured stars expected to come off the IL soon -- or have already -- could rock MLB's playoff races.7hAlden GonzalezJay Biggerstaff-USA TODAY Sports\\n\\n'CLEARLY THE ACC IS STRUGGLING'\", metadata={'source': 'https://www.espn.com/'}),\n",
+       " Document(page_content=\"'CLEARLY THE ACC IS STRUGGLING'\\n\\nFinebaum: FSU is better off leaving the ACC\\n\\n5h\\n\\n1:04\\n\\nThamel's realignment buzz: Latest on Pac-12, Big 12 and ACC\\n\\nAN AGGRESSIVE STRATEGY\\n\\nHow the Big 12 landed Colorado and shook up college footballThe Big 12 learned lessons two years ago after getting burned by Texas and Oklahoma. It resulted in a more aggressive strategy that could dramatically change the sport.2dHeather DinichRaymond Carlin/Icon Sportswire\\n\\nTop Headlines\\n\\nFreeze 'uncomfortable' as Auburn opens camp\\n\\nTexans' Metchie relied on faith amid cancer fight\\n\\nHornets have new owners after MJ sale finalized\\n\\nMiami coach expects rough treatment of Messi\\n\\nDrexel basketball player found dead in apartment\\n\\nGermany exits WWC after draw with South Korea\\n\\nBrady takes minority stake in English soccer team\\n\\nDeep dish: Cubs' output at plate best since 1897\\n\\nRe-drafting 2018 NFL class 5 years later\\n\\nFavorites\\n\\nFantasy\\n\\nManage Favorites\\n\\nFantasy Home\\n\\nCustomize ESPN\\n\\nICYMI\\n\\n0:54\\n\\nSerena Williams, Alexis Ohanian use drones to reveal gender of 2nd childSerena Williams and her husband Alexis Ohanian find out the gender of their second child in a spectacular display of drones.\\n\\nBest of ESPN+\\n\\nTodd Kirkland/Getty Images\\n\\nMLB 2023 trade deadline: Winners, losers and in-betweenersThe 2023 trade deadline is over! Who crushed it, and who left much to be desired? We weigh in on all 30 clubs.\\n\\nAP Photo/Matt York\", metadata={'source': 'https://www.espn.com/'}),\n",
+       " Document(page_content='AP Photo/Matt York\\n\\nLowe: Why Bradley Beal could unlock KD, Book and the most dangerous version of the Phoenix Suns yetWith Kevin Durant, Devin Booker and Beal, Phoenix is already an inner-circle title contender. But if the Suns continue a Beal experiment the Wizards ran last season? Good luck.\\n\\nCliff Welch/Icon Sportswire\\n\\nPredicting 10 NFL starting quarterback battles: Who is QB1?We talked to people around the NFL and projected the QB1 for 10 unsettled situations, including a wide-open race in Tampa Bay.\\n\\nTrending Now\\n\\nAP Photo/Julio Cortez\\n\\n\\'Revis Island\\' resonates long after Hall of Famer\\'s retirementDarrelle Revis made his name as a dominant corner but might be best known for his \"island\" moniker players still adopt today.\\n\\nIllustration by ESPN\\n\\nThe wild life of Gardner MinshewFour colleges, three NFL teams, two Manias and the hug that broke the internet. It\\'s been an unbelievable ride for Gardner Minshew. Next stop: Indianapolis.\\n\\nIllustration by ESPN\\n\\nBest 2023 Women\\'s World Cup players: Morgan, Caicedo, moreESPN\\'s expert panel selected the top 25 players of the Women\\'s World Cup to keep an eye on, from Sophia Smith to Sam Kerr and more.\\n\\nHow to Watch on ESPN+\\n\\n(AP Photo/Koji Sasahara, File)\\n\\nHow to watch the PGA Tour, Masters, PGA Championship and FedEx Cup playoffs on ESPN, ESPN+Here\\'s everything you need to know about how to watch the PGA Tour, Masters, PGA Championship and FedEx Cup playoffs on ESPN and ESPN+.\\n\\nSign up to play the #1 Fantasy game!', metadata={'source': 'https://www.espn.com/'}),\n",
+       " Document(page_content=\"Sign up to play the #1 Fantasy game!\\n\\nCreate A League\\n\\nJoin Public League\\n\\nReactivate\\n\\nMock Draft Now\\n\\nSign up for FREE!\\n\\nCreate A League\\n\\nJoin a Public League\\n\\nReactivate a League\\n\\nPractice With a Mock Draft\\n\\nSign up for FREE!\\n\\nCreate A League\\n\\nJoin a Public League\\n\\nReactivate a League\\n\\nPractice with a Mock Draft\\n\\nESPN+\\n\\nWatch Now\\n\\nPaul vs. Diaz (ESPN+ PPV)\\n\\nPGA TOUR LIVE\\n\\nLittle League Baseball: Regionals\\n\\nMLB: Select Games\\n\\nCrossFit Games\\n\\nSlamBall\\n\\nThe Ultimate Fighter: Season 31\\n\\nFantasy Football: Top Storylines, Rookies, Sleepers\\n\\nQuick Links\\n\\nWomen's World Cup\\n\\nNHL Free Agency\\n\\nNBA Free Agency Buzz\\n\\nNBA Trade Machine\\n\\nThe Basketball Tournament\\n\\nFantasy Football: Sign Up\\n\\nHow To Watch PGA TOUR\\n\\nESPN Sites\\n\\nESPN Deportes\\n\\nAndscape\\n\\nespnW\\n\\nESPNFC\\n\\nX Games\\n\\nSEC Network\\n\\nESPN Apps\\n\\nESPN\\n\\nESPN Fantasy\\n\\nFollow ESPN\\n\\nFacebook\\n\\nX/Twitter\\n\\nInstagram\\n\\nSnapchat\\n\\nTikTok\\n\\nYouTube\\n\\nTerms of Use\\n\\nPrivacy Policy\\n\\nYour US State Privacy Rights\\n\\nChildren's Online Privacy Policy\\n\\nInterest-Based Ads\\n\\nAbout Nielsen Measurement\\n\\nDo Not Sell or Share My Personal Information\\n\\nContact Us\\n\\nDisney Ad Sales Site\\n\\nWork for ESPN\", metadata={'source': 'https://www.espn.com/'}),\n",
+       " Document(page_content='Skip to main content\\n\\nlink\\n\\nHome\\n\\nPeople\\n\\nMinor\\n\\n\\tMinor Requirements\\n\\tCourse Descriptions\\n\\tCourse Schedule\\n\\tHow to Declare the Minor\\n\\tChoosing a Minor\\n\\nResearch and Immersion\\n\\n\\tResearch and Immersion Overview\\n\\tDS 3850 Research in Data Science\\n\\tDSI Summer Research Program\\n\\tData Science for Social Good\\n\\tResearch Immersion in Data Science\\n\\tDSI Internship\\n\\nFAQ\\n\\nNews\\n\\nForms\\n\\nContact and Email List\\n\\nData Science Institute\\n\\nUndergraduate Data Science\\n\\nFrequently Asked Questions\\n\\nDeclaring the Minor\\n\\nHow do I declare the Data Science Minor?\\n\\nUse the forms and follow the procedures for your home college. See How to Declare the Data Science Minor.\\n\\nWhen should I declare the Data Science Minor?\\n\\nWhile minor declarations can be made any time, DS courses will give some preference to students who have officially declared the Data Science Minor. So we recommend declaring the minor sooner rather than later. It is always possible to drop a declared minor. Minor declarations must be submitted at least two weeks before registration begins. Otherwise, the minor declaration will not be processed until after registration. No preference will be given during registration for an “intent” to declare because the minor declaration was made too late.\\n\\nI declared the Data Science Minor, but I did not get into the class I wanted to take for the minor. Why?', metadata={'source': 'https://www.vanderbilt.edu/undergrad-datascience/faq'}),\n",
+       " Document(page_content='I declared the Data Science Minor, but I did not get into the class I wanted to take for the minor. Why?\\n\\nFirst, preference for students who have declared the minor only applies to DS courses, not other courses. Second, if you declared the minor within two weeks of registration, your minor declaration will. not show up on YES, and you will not have preference. Third, while we try to hold as many seats for students who have declared the minor as we can, not all seats are reserved.\\n\\nI am a first-year A&S student. Can I really declare the Data Science Minor now?\\n\\nYes. While A&S students are usually prevented from declaring a major or minor until sophomore year, first-year A&S students can declare the Data Science Minor. As noted in the previous question, this can be important to do since some popular core DS courses will give some preference to students who have officially declared Data Science as a minor.\\n\\nI am a current junior (rising senior), can I complete the Data Science Minor (for Spring 2021 juniors only)?', metadata={'source': 'https://www.vanderbilt.edu/undergrad-datascience/faq'}),\n",
+       " Document(page_content='I am a current junior (rising senior), can I complete the Data Science Minor (for Spring 2021 juniors only)?\\n\\nJuniors must contact the Director of Undergraduate Data Science to discuss options. DS 1000 is not open to current juniors (rising seniors). DS 3100 will not be taught next year (Fall 2021 or Spring 2022) and will need to be suitably replaced, which will require an approved plan from the Director. Furthermore, while DS / CS 3262 is current slated to be taught Spring 2022, that is not fully guaranteed, so students should see if they can take one of the other machine learning options.\\n\\nI am a rising senior or current senior and cannot register for DS 1000. Why?\\n\\nRising seniors and current seniors can only register for DS 1000 if there are available seats immediately before the semester begins with permission of the instructor. DS 1000 is intended as an introduction to data science for first years and sophomores, which is why this restriction is in place.\\n\\nCollege-Specific Information\\n\\nWhat college is the home of the Data Science Minor?\\n\\nThe Data Science Minor is a trans-institutional minor, shared by A&S, Blair, Engineering, and Peabody.\\n\\nI am an A&S student. Do DS courses count as A&S courses?', metadata={'source': 'https://www.vanderbilt.edu/undergrad-datascience/faq'}),\n",
+       " Document(page_content='I am an A&S student. Do DS courses count as A&S courses?\\n\\nAll courses with a DS prefix count as courses within each of the colleges, including A&S. If you are an A&S student, and are taking a course that is cross-listed, make sure you enroll in the one with the DS prefix. Electives outside of A&S without the DS prefix will generally not count as A&S courses, so plan accordingly.\\n\\nWhat are the unique credit hour rules for the Data Science Minor?\\n\\nStudents electing an undergraduate minor in Data Science must follow academic regulations regarding minors in their home college, including but not limited to regulations regarding unique hours. The unique credit hour rule is specific to the College of Arts and Science and Peabody College. The School of Engineering and Blair School of Music do not have a unique credit hour rule. The Data Science minor cannot waive this rule. Please talk with your academic advisor about how to satisfy these requirements.\\n\\nInfo About the Courses\\n\\nDS 1000\\n\\nThank you for your interest in DS 1000! The course is full for the fall 2021 semester. Due to student demand and the transinstitutional nature of the course, we cannot make special exceptions as to which students, if any, on the waitlist are able to enroll. DS 1000 will be offered again in the spring semester.\\n\\nWhat computer programming course should I take?', metadata={'source': 'https://www.vanderbilt.edu/undergrad-datascience/faq'}),\n",
+       " Document(page_content='What computer programming course should I take?\\n\\nSee What Programming Course To Take? In general, students interested in data science and scientific computing (not in computer science per se) should learn Python (and R).\\n\\nHow do I find courses approved for the data science minor on YES?\\n\\nOn YES, to select all courses approved for credit in the Data Science minor offered in a given semester, select the “Advanced” link next to the search box, select the “Class Attributes” drop-down box on the bottom right of the advanced search page, and then select “Eligible for Data Science” to find all courses. (Note that these course tags will not all be in place on YES until the registration period for Fall 2021 begins.)\\n\\nCan other courses, besides those listed, count towards the Data Science Minor?\\n\\nNew courses, special topics courses, or graduate-level courses that seem related to data science could count as electives. Contact the Director of Undergraduate Data Science to request consideration.\\n\\nWhy doesn’t CS 1104 count towards the Data Science Minor?', metadata={'source': 'https://www.vanderbilt.edu/undergrad-datascience/faq'}),\n",
+       " Document(page_content='Why doesn’t CS 1104 count towards the Data Science Minor?\\n\\nIt does, as a prerequisite to CS 2204, which counts towards the minor. CS / DS 1100 was created as a new single-semester programming course for the Data Science Minor. It roughly has 2/3 the content of CS 1104 and 1/3 the content of CS 2204. While CS / DS 1100 counts as a single semester of programming for the minor, we strongly encourage students interested in data science, and in using data science tools and techniques, to take two semesters of programming in Python (CS / DS 1100 or CS 1104, followed by CS 2204). If you have taken CS 1104, you can take CS 1100, but you will only receive a total of four credits for\\xa0the two courses. See also What Programming Course To Take?\\n\\nI see that after having taken CS 1104, I can take CS/DS 1100 instead of taking CS 2204. What are the downsides of doing so?', metadata={'source': 'https://www.vanderbilt.edu/undergrad-datascience/faq'}),\n",
+       " Document(page_content='I see that after having taken CS 1104, I can take CS/DS 1100 instead of taking CS 2204. What are the downsides of doing so?\\n\\nAfter taking CS 1104, we do recommend you take CS 2204. If you are interested in data science, a broader experience in Python in desirable (in fact, we recommend that students having taken CS 1100 try to take CS 2204 as well). CS/DS 1100 and 1104 have significant overlap (both are introductions to programming using Python). That said, it is permissible to take CS/DS 1100 after having taken CS 1104. You will only get 1 (out of 3) credit hours for CS/DS 1100 (after having taken CS 1104), but the combination of CS/DS 1100 and 1104 will satisfy the DS minor programming requirement. Note that if you enroll in three 3-hour courses and CS/DS 1100 (after having taken CS 1104) it will look like you are registered for 12 credit hours during registration and at the start of the semester, but your credit hours will be reduced to only 10 credit hours (because the credits for CS/DS 1100 will be cut back to 1 after the add/drop period). Enrolling in fewer than 12 credit hours can have significant consequences on financial aid and potentially on visa status for international students. Please be mindful of this.\\n\\nWhat is the difference between CS 1100 and DS 1100?\\n\\nNothing. They are the same course. They meet the same time in the same place and are taught by the same instructor. They are just cross-listed.', metadata={'source': 'https://www.vanderbilt.edu/undergrad-datascience/faq'}),\n",
+       " Document(page_content='Nothing. They are the same course. They meet the same time in the same place and are taught by the same instructor. They are just cross-listed.\\n\\nI have taken CS 1101. What computer programming course should I take next?\\n\\nYou have two options. You can either take CS 2201 (in C++) or take CS 1100 (in Python). Of course, you could also take CS 1104 and 2204 (in Python). CS 1100, 2201, and 2204 all satisfy the programming requirement for the minor. Note that CS 2201 is a prerequisite for many upper-level CS courses (as well as required for the CS major and minor). For more information, see What Programming Course To Take?\\n\\nECON 3750 and MATH 3670 are listed both as satisfying the core machine learning requirement and as electives. If I take one, will it double-count for both requirements?\\n\\nNo. They are listed under both because a student who takes one of the other machine learning\\xa0courses to satisfy the core requirement (CS/DS 3262 or CS 4262) can also take ECON 3750 or MATH 3670 as an elective; the content is sufficiently different that both can count towards the minor, but one course cannot double-count for two minor requirements.\\n\\nCan I take ECON 3750 or MATH 3670 as an elective if I have already taken CS 3262 or CS 4262?', metadata={'source': 'https://www.vanderbilt.edu/undergrad-datascience/faq'}),\n",
+       " Document(page_content='Can I take ECON 3750 or MATH 3670 as an elective if I have already taken CS 3262 or CS 4262?\\n\\nYes (see above). ECON 3750 and MATH 3670 are sufficiently different from CS 3262 or CS 4262 (and from each other) that you can take these as electives. In fact, you could take ECON 3750 to satisfy the machine learning requirement and then take MATH 3670 as an elective.\\n\\nCS 3262 can count towards the Data Science minor. CS 3262 does not count directly towards the Computer Science major requirements but could be used as either a tech elective or open elective for Computer Science majors.\\n\\nWhy doesn’t MATH 2820 count towards the Data Science Minor?\\n\\nIt does, as a prerequisite to MATH 2821, which counts towards the minor. The two-course sequence of MATH 2820 and MATH 2821 counts towards the Data Science Minor; the\\xa0two-course sequence is required because MATH 2820 goes deep into mathematical foundations of probability ad statistics concepts, but does not by itself cover the breadth of topics of other introductory statistics courses. This two-course sequence provides an excellent introduction to mathematical statistics.\\n\\nResearch and Immersion Information\\n\\nCan I do research for course credit?\\n\\nYes, you can do research for course credit (including DS 3850). More information can be found here: https://www.vanderbilt.edu/undergrad-datascience/ds-3850-research-in-data-science/\\n\\nI am interested in the Undergraduate Data Science Immersion Program. How can I participate.', metadata={'source': 'https://www.vanderbilt.edu/undergrad-datascience/faq'}),\n",
+       " Document(page_content='I am interested in the Undergraduate Data Science Immersion Program. How can I participate.\\n\\nSome competitive summer immersion programs include DSI-SPR and Data Science for Social Good (DSSG). More information can be found on the following websites.\\n\\nhttps://www.vanderbilt.edu/datascience/academics/undergraduate/summer-research-program/\\n\\nhttps://www.vanderbilt.edu/datascience/data-science-for-social-good/\\n\\nTo get involved in data-science-oriented research with a faculty member, you will need to reach out to the faculty member. Pointers can be found here: https://www.vanderbilt.edu/undergrad-datascience/research-and-immersion-overview/. Having that research count towards the immersion requirement will be between your faculty mentor and your faculty immersion coordinator.\\n\\nAdditional information about research opportunities will be posted on the website in the future.\\n\\nContact\\n\\nHow do I ask a question about the Data Science?\\n\\nIf you have questions about the Data Science Minor or Immersion opportunities in data science, please email us: [email protected]\\n\\nTo whom can I petition if the Director denies my request?\\n\\nThe Governing Board of the Data Science Minor acts as the college-level oversight body for this trans-institutional minor and would be the appropriate next step for petitions related to the minor.\\n\\nData Science News\\n\\nOpportunities for Capstone Projects and Research Experience\\n\\nAttention Graduate Students! We’re Hiring!', metadata={'source': 'https://www.vanderbilt.edu/undergrad-datascience/faq'}),\n",
+       " Document(page_content='Data Science News\\n\\nOpportunities for Capstone Projects and Research Experience\\n\\nAttention Graduate Students! We’re Hiring!\\n\\nVanderbilt student-athlete drives sports performance through data analysis\\n\\nNew Course: DS 3891 Special Topics: Intro to Generative AI\\n\\nNow Accepting Applications: DS Minor Teaching Fellowship for graduate students\\n\\nJoin Our Team: Student Worker Positions Available for Fall 2023 Semester!\\n\\nVIEW MORE EVENTS >\\n\\nYour Vanderbilt\\n\\nAlumni\\n\\nCurrent Students\\n\\nFaculty & Staff\\n\\nInternational Students\\n\\nMedia\\n\\nParents & Family\\n\\nProspective Students\\n\\nResearchers\\n\\nSports Fans\\n\\nVisitors & Neighbors\\n\\nQuick Links\\n\\nPeopleFinder\\n\\nLibraries\\n\\nNews\\n\\nCalendar\\n\\nMaps\\n\\nA-Z\\n\\n©\\n                    Site Development: Digital Strategies (Division of Communications)\\n                    Vanderbilt University is committed to principles of equal opportunity and affirmative action. Accessibility information. Vanderbilt®, Vanderbilt University®, V Oak Leaf Design®, Star V Design® and Anchor Down® are trademarks of The Vanderbilt University', metadata={'source': 'https://www.vanderbilt.edu/undergrad-datascience/faq'})]"
+      ]
+     },
+     "execution_count": null,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# get the results\n",
+    "res_unstructured = website_to_text_unstructured(demo_urls)\n",
+    "res_unstructured"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#unit testb\n",
+    "test_converters_inputs(website_to_text_unstructured, demo_urls)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "We also see here that there's something to be said about the unstructured approach which appears to be more conservative in the number of newline characters but still appears to preserve content. However, the gain is not overly significant.\n",
+    "\n",
+    "#### Trafilatura Parsing\n",
+    "\n",
+    "[Tralifatura](https://trafilatura.readthedocs.io/en/latest/) is a Python and command-line utility which attempts to extracts the most relevant information from a given website.  "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def website_trafilatura(url):\n",
+    "  downloaded = trafilatura.fetch_url(url)\n",
+    "  return trafilatura.extract(downloaded)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Total number of characters in example: 1565 \n",
+      "\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "'|\\n|\\n|\\n|\\n|\\n||\\n|\\n|\\n||\\nPHI\\nMIA\\n||\\n56-49\\n57-49\\n||\\n||\\n||\\n||\\n||\\n6:40 PM ET\\n||\\n|\\n|\\n|\\n|\\n|\\n|\\n||\\n|\\n|\\n||\\nMIL\\nWSH\\n||\\n57-49\\n44-62\\n||\\n||\\n||\\n||\\n||\\n7:05 PM ET\\n||\\n|\\n|\\n|\\n|\\n|\\n|\\n||\\n|\\n|\\n||\\nTB\\nNYY\\n||\\n64-44\\n55-50\\n||\\n||\\n||\\n||\\n||\\n7:05 PM ET\\n||\\n|\\n|\\n|\\n|\\n|\\n|\\n||\\n|\\n|\\n||\\nBAL\\nTOR\\n||\\n64-41\\n59-47\\n||\\n||\\n||\\n||\\n||\\n7:07 PM ET\\n||\\n|\\n|\\n|\\n|\\n|\\n|\\n||\\n|\\n|\\n||\\nLAA\\nATL\\n||\\n55-51\\n67-36\\n||\\n||\\n||\\n||\\n||\\n7:20 PM ET\\n||\\n|\\n|\\n|\\n|\\n|\\n|\\n||\\n|\\n|\\n||\\nCIN\\nCHC\\n||\\n58-49\\n53-52\\n||\\n||\\n||\\n||\\n||\\n8:05 PM ET\\n||\\n|\\n|\\n|\\n|\\n|\\n|\\n||\\n|\\n|\\n||\\nCLE\\nHOU\\n||\\n53-53\\n59-47\\n||\\n||\\n||\\n||\\n||\\n8:10 PM ET\\n||\\n|\\n|\\n|\\n|\\n|\\n|\\n||\\n|\\n|\\n||\\nSD\\nCOL\\n||\\n52-54\\n41-64\\n||\\n||\\n||\\n||\\n||\\n8:40 PM ET\\n||\\n|\\n|\\n|\\n|\\n|\\n|\\n||\\n|\\n|\\n||\\nBOS\\nSEA\\n||\\n56-49\\n54-51\\n||\\n||\\n||\\n||\\n||\\n9:40 PM ET\\n||\\n|\\n|\\n|\\n|\\n|\\n|\\n||\\n|\\n|\\n||\\nARI\\nSF\\n||\\n56-50\\n58-48\\n||\\n||\\n||\\n||\\n||\\n9:45 PM ET\\n||\\n|\\n|\\n|\\n|\\n|\\n|\\n||\\n|\\n|\\n||\\nJPN\\nESP\\n||\\n4\\n0\\n||\\n||\\n||\\n||\\n||\\nFT\\n||\\n|\\n|\\n|\\n|\\n|\\n|\\n||\\n|\\n|\\n||\\nCRC\\nZAM\\n||\\n1\\n3\\n||\\n||\\n||\\n||\\n||\\nFT\\n||\\n|\\n|\\n|\\n|\\n|\\n|\\n||\\n|\\n|\\n||\\nCAN\\nAUS\\n||\\n0\\n4\\n||\\n||\\n||\\n||\\n||\\nFT\\n||\\n|\\n|\\n|\\n|\\n|\\n|\\n||\\n|\\n|\\n||\\nIRL\\nNGA\\n||\\n0\\n0\\n||\\n||\\n||\\n||\\n||\\nFT\\n||\\n|\\n|\\n|\\n|\\n|\\n|\\n||\\n|\\n|\\n||\\nPOR\\nUSA\\n||\\nWLDDL\\nDWWWW\\n||\\n||\\n||\\n||\\n||\\n3:00 AM ET\\n||\\n|\\n|\\n|\\n|\\n|\\n|\\n||\\n|\\n|\\n||\\nVIE\\nNED\\n||\\nLLLLL\\nDWWWL\\n||\\n||\\n||\\n||\\n||\\n3:00 AM ET\\n||\\n|\\n|\\n|\\n|\\n|\\n|\\n||\\n|\\n|\\n||\\nCHN\\nENG\\n||\\nWLDWW\\nWWDLW\\n||\\n||\\n||\\n||\\n||\\n7:00 AM ET\\n||\\n|\\n|\\n|\\n|\\n|\\n|\\n||\\n|\\n|\\n||\\nHAI\\nDEN\\n||\\nLLLWL\\nLWLWW\\n||\\n||\\n||\\n||\\n||\\n7:00 AM ET\\n||\\n|\\n|\\n|\\n|\\n|\\n|\\n||\\n|\\n|\\n||\\nAME\\nCLB\\n||\\nWWLLW\\nWLDDW\\n||\\n||\\n||\\n||\\n||\\n8:00 PM ET\\n||\\n|\\n|\\n|\\n|\\n|\\n|\\n||\\n|\\n|\\n||\\nPUE\\nCHI\\n||\\nLLLDL\\nWWWWL\\n||\\n||\\n||\\n||\\n||\\n8:00 PM ET\\n||\\n|\\n|\\n|\\n|\\n|\\n|\\n||\\n|\\n|\\n||\\nTOL\\nCOL\\n||\\nWLWDW\\nLDDWL\\n||\\n||\\n||\\n||\\n||\\n9:30 PM ET\\n||\\n|\\n|\\n|\\n|\\n|\\n|\\n||\\n|\\n|\\n||\\nGDL\\nSKC\\n||\\nLWWWL\\nLLDDW\\n||\\n||\\n||\\n||\\n||\\n10:00 PM ET\\n||\\n|\\n|\\n|'"
+      ]
+     },
+     "execution_count": null,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "trafilatura_text = website_trafilatura(demo_urls[0])\n",
+    "print('Total number of characters in example:', len(trafilatura_text), '\\n')\n",
+    "trafilatura_text"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "This output is SUBSTANTIALLY shorter with a length of 1565 characters. However, the problem is that the main article on the page actually isn't captured at all.\n",
+    "\n",
+    "#### jusText\n",
+    "\n",
+    "[jusText](https://pypi.org/project/jusText/) is another Python library for extracting content from a website."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def website_justext(url):\n",
+    "  response = requests.get(url)\n",
+    "  paragraphs = justext.justext(response.content, justext.get_stoplist(\"English\"))\n",
+    "  content = [paragraph.text for paragraph in paragraphs \\\n",
+    "            if not paragraph.is_boilerplate]\n",
+    "  text = \" \".join(content)\n",
+    "  return text"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "''"
+      ]
+     },
+     "execution_count": null,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Ensure behavior\n",
+    "justext_text = website_justext(demo_urls[0])\n",
+    "justext_text"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'Declaring the Minor While minor declarations can be made any time, DS courses will give some preference to students who have officially declared the Data Science Minor. So we recommend declaring the minor sooner rather than later. It is always possible to drop a declared minor. Minor declarations must be submitted at least two weeks before registration begins. Otherwise, the minor declaration will not be processed until after registration. No preference will be given during registration for an “intent” to declare because the minor declaration was made too late. First, preference for students who have declared the minor only applies to DS courses, not other courses. Second, if you declared the minor within two weeks of registration, your minor declaration will. not show up on YES, and you will not have preference. Third, while we try to hold as many seats for students who have declared the minor as we can, not all seats are reserved. Yes. While A&S students are usually prevented from declaring a major or minor until sophomore year, first-year A&S students can declare the Data Science Minor. As noted in the previous question, this can be important to do since some popular core DS courses will give some preference to students who have officially declared Data Science as a minor. Juniors must contact the Director of Undergraduate Data Science to discuss options. DS 1000 is not open to current juniors (rising seniors). DS 3100 will not be taught next year (Fall 2021 or Spring 2022) and will need to be suitably replaced, which will require an approved plan from the Director. Furthermore, while DS / CS 3262 is current slated to be taught Spring 2022, that is not fully guaranteed, so students should see if they can take one of the other machine learning options. Rising seniors and current seniors can only register for DS 1000 if there are available seats immediately before the semester begins with permission of the instructor. DS 1000 is intended as an introduction to data science for first years and sophomores, which is why this restriction is in place. All courses with a DS prefix count as courses within each of the colleges, including A&S. If you are an A&S student, and are taking a course that is cross-listed, make sure you enroll in the one with the DS prefix. Electives outside of A&S without the DS prefix will generally not count as A&S courses, so plan accordingly. Students electing an undergraduate minor in Data Science must follow academic regulations regarding minors in their home college, including but not limited to regulations regarding unique hours. The unique credit hour rule is specific to the College of Arts and Science and Peabody College. The School of Engineering and Blair School of Music do not have a unique credit hour rule. The Data Science minor cannot waive this rule. Please talk with your academic advisor about how to satisfy these requirements. Info About the Courses Thank you for your interest in DS 1000! The course is full for the fall 2021 semester. Due to student demand and the transinstitutional nature of the course, we cannot make special exceptions as to which students, if any, on the waitlist are able to enroll. DS 1000 will be offered again in the spring semester. On YES, to select all courses approved for credit in the Data Science minor offered in a given semester, select the “Advanced” link next to the search box, select the “Class Attributes” drop-down box on the bottom right of the advanced search page, and then select “Eligible for Data Science” to find all courses. (Note that these course tags will not all be in place on YES until the registration period for Fall 2021 begins.) It does, as a prerequisite to CS 2204, which counts towards the minor. CS / DS 1100 was created as a new single-semester programming course for the Data Science Minor. It roughly has 2/3 the content of CS 1104 and 1/3 the content of CS 2204. While CS / DS 1100 counts as a single semester of programming for the minor, we strongly encourage students interested in data science, and in using data science tools and techniques, to take two semesters of programming in Python (CS / DS 1100 or CS 1104, followed by CS 2204). If you have taken CS 1104, you can take CS 1100, but you will only receive a total of four credits for the two courses. See also What Programming Course To Take? After taking CS 1104, we do recommend you take CS 2204. If you are interested in data science, a broader experience in Python in desirable (in fact, we recommend that students having taken CS 1100 try to take CS 2204 as well). CS/DS 1100 and 1104 have significant overlap (both are introductions to programming using Python). That said, it is permissible to take CS/DS 1100 after having taken CS 1104. You will only get 1 (out of 3) credit hours for CS/DS 1100 (after having taken CS 1104), but the combination of CS/DS 1100 and 1104 will satisfy the DS minor programming requirement. Note that if you enroll in three 3-hour courses and CS/DS 1100 (after having taken CS 1104) it will look like you are registered for 12 credit hours during registration and at the start of the semester, but your credit hours will be reduced to only 10 credit hours (because the credits for CS/DS 1100 will be cut back to 1 after the add/drop period). Enrolling in fewer than 12 credit hours can have significant consequences on financial aid and potentially on visa status for international students. Please be mindful of this. You have two options. You can either take CS 2201 (in C++) or take CS 1100 (in Python). Of course, you could also take CS 1104 and 2204 (in Python). CS 1100, 2201, and 2204 all satisfy the programming requirement for the minor. Note that CS 2201 is a prerequisite for many upper-level CS courses (as well as required for the CS major and minor). For more information, see What Programming Course To Take? No. They are listed under both because a student who takes one of the other machine learning courses to satisfy the core requirement (CS/DS 3262 or CS 4262) can also take ECON 3750 or MATH 3670 as an elective; the content is sufficiently different that both can count towards the minor, but one course cannot double-count for two minor requirements. Yes (see above). ECON 3750 and MATH 3670 are sufficiently different from CS 3262 or CS 4262 (and from each other) that you can take these as electives. In fact, you could take ECON 3750 to satisfy the machine learning requirement and then take MATH 3670 as an elective. CS 3262 can count towards the Data Science minor. CS 3262 does not count directly towards the Computer Science major requirements but could be used as either a tech elective or open elective for Computer Science majors. It does, as a prerequisite to MATH 2821, which counts towards the minor. The two-course sequence of MATH 2820 and MATH 2821 counts towards the Data Science Minor; the two-course sequence is required because MATH 2820 goes deep into mathematical foundations of probability ad statistics concepts, but does not by itself cover the breadth of topics of other introductory statistics courses. This two-course sequence provides an excellent introduction to mathematical statistics.'"
+      ]
+     },
+     "execution_count": null,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Try a different URL to see if behavior improves\n",
+    "justext_text = website_justext(demo_urls[1])\n",
+    "justext_text"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Here, we see that we may prefer to stick with the langchain implementations. The first jusText example returned an empty string, although previous work demonstrates that on a different day, it worked well (note that the ESPN's content was different). With the second URL, parts of the website, particularly the headers, is actually missing."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Creating Document Segments\n",
+    "Now, the precursor to creating vector stores/embeddings is to create document segments. Since we have a variety of sources, we will keep this in mind as we develop the following function.\n",
+    "\n",
+    ":::{.callout-warning}\n",
+    "Note that the `get_document_segments` currently is meant to be used in one single pass with `context_info` being all of a single file type. [Issue #150](https://github.com/vanderbilt-data-science/lo-achievement/issues/150) is meant to expand this functionality so that if many files are uploaded, the software will be able to handle this.\n",
+    ":::"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#| export\n",
+    "def get_document_segments(context_info, data_type, chunk_size = 1500, chunk_overlap=100):\n",
+    "\n",
+    "    load_fcn = None\n",
+    "    addtnl_params = {'chunk_size': chunk_size, 'chunk_overlap': chunk_overlap}\n",
+    "\n",
+    "    # Define function use to do the loading\n",
+    "    if data_type == 'text':\n",
+    "        load_fcn = rawtext_to_doc_split\n",
+    "    elif data_type == 'web_page':\n",
+    "        load_fcn = website_to_text_unstructured\n",
+    "    elif data_type == 'youtube_video':\n",
+    "        load_fcn = youtube_to_text\n",
+    "    else:\n",
+    "        load_fcn = files_to_text\n",
+    "    \n",
+    "    # Get the document segments\n",
+    "    doc_segments = load_fcn(context_info, **addtnl_params)\n",
+    "\n",
+    "    return doc_segments"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Creating Vector Stores from Document Segments\n",
+    "The last step here will be in the creation of vector stores from the provided document segments. We will allow for the usage of either Chroma or DeepLake and enforce OpenAIEmbeddings."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#| export\n",
+    "def create_local_vector_store(document_segments, **retriever_kwargs):\n",
+    "    embeddings = OpenAIEmbeddings()\n",
+    "    db = Chroma.from_documents(document_segments, embeddings)\n",
+    "    retriever = db.as_retriever(**retriever_kwargs)\n",
+    "    \n",
+    "    return db, retriever"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Unit test of vector store and segment creation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.chat_models import ChatOpenAI\n",
+    "from getpass import getpass"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "openai_api_key = getpass()\n",
+    "os.environ[\"OPENAI_API_KEY\"] = openai_api_key\n",
+    "\n",
+    "llm = ChatOpenAI(model_name = 'gpt-3.5-turbo-16k')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "test_files = ['../roadnottaken.txt', '../2302.11382.pdf']\n",
+    "\n",
+    "#get vector store\n",
+    "segs = get_document_segments(test_files, data_type='other', chunk_size = 1000, chunk_overlap = 100)\n",
+    "chroma_db, vs_retriever = create_local_vector_store(segs)\n",
+    "\n",
+    "#create test retrievalqa\n",
+    "qa_chain = RetrievalQA.from_chain_type(llm=openai_llm, chain_type=\"stuff\", retriever=vs_retriever)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[Document(page_content='Two roads diverged in a yellow wood,\\rAnd sorry I could not travel both\\rAnd be one traveler, long I stood\\rAnd looked down one as far as I could\\rTo where it bent in the undergrowth;\\r\\rThen took the other, as just as fair,\\rAnd having perhaps the better claim,\\rBecause it was grassy and wanted wear;\\rThough as for that the passing there\\rHad worn them really about the same,\\r\\rAnd both that morning equally lay\\rIn leaves no step had trodden black. Oh, I kept the first for another day! Yet knowing how way leads on to way,\\rI doubted if I should ever come back. I shall be telling this with a sigh\\rSomewhere ages and ages hence:\\rTwo roads diverged in a wood, and IэI took the one less traveled by,\\rAnd that has made all the difference.', metadata={'source': '../roadnottaken.txt', 'start_index': 0}),\n",
+       " Document(page_content='any unnecessary steps,” is useful in flagging inaccuracies in the user’s original request so that the final recipe is efficient.', metadata={'source': '../2302.11382.pdf', 'start_index': 92662}),\n",
+       " Document(page_content='The third statement provides an optional way for the user to stop the output generation process. This step is not always needed, but can be useful in situations where there may be the potential for ambiguity regarding whether or not the user- provided input between inputs is meant as a refinement for the next generation or a command to stop. For example, an explicit stop phrase could be created if the user was generating data related to road signs, where the user might want to enter a refinement of the generation like “stop” to indicate that a stop sign should be added to the output.', metadata={'source': '../2302.11382.pdf', 'start_index': 72043}),\n",
+       " Document(page_content='“When I ask you a question, generate three addi- tional questions that would help you give a more accurate answer. Assume that I know little about the topic that we are discussing and please define any terms that are not general knowledge. When I have answered the three questions, combine the answers to produce the final answers to my original question.”\\n\\nOne point of variation in this pattern is where the facts are output. Given that the facts may be terms that the user is not familiar with, it is preferable if the list of facts comes after the output. This after-output presentation ordering allows the user to read and understand the statements before seeing what statements should be checked. The user may also determine additional facts prior to realizing the fact list at the end should be checked.', metadata={'source': '../2302.11382.pdf', 'start_index': 57473})]"
+      ]
+     },
+     "execution_count": null,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# check for functionality\n",
+    "chroma_db.similarity_search('The street was forked and I did not know which way to go')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#check qa chain for functionality\n",
+    "ans = qa_chain({'question':'What is the best prompt to use when I want the model to take on a certain attitude of a person?'})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'question': 'What is the best prompt to use when I want the model to take on a certain attitude of a person?',\n",
+       " 'answer': 'The best prompt to use when you want the model to take on a certain attitude of a person is to provide a persona for the model to embody. This can be expressed as a job description, title, fictional character, historical figure, or any other attributes associated with a well-known type of person. The prompt should specify the outputs that this persona would create. Additionally, personas can also represent inanimate or non-human entities, such as a Linux terminal or a database. In this case, the prompt should specify how the inputs should be delivered to the entity and what outputs the entity should produce. It is also possible to provide a better version of the question and prompt the model to ask if the user would like to use the better version instead.\\n',\n",
+       " 'sources': '../2302.11382.pdf',\n",
+       " 'source_documents': [Document(page_content='4) Example Implementation: A sample prompt for a flipped\\n\\ninteraction is shown below:\\n\\n“From now on, I would like you to ask me questions to deploy a Python application to AWS. When you have enough information to deploy the application, create a Python script to automate the deployment.”\\n\\n2) Motivation: Users may not know what types of outputs or details are important for an LLM to focus on to achieve a given task. They may know, however, the role or type of person that they would normally ask to get help with these things. The Persona pattern enables the users to express what they need help with without knowing the exact details of the outputs they need.', metadata={'source': '../2302.11382.pdf', 'start_index': 36397}),\n",
+       "  Document(page_content='ments:\\n\\nContextual Statements Act as persona X Provide outputs that persona X would create\\n\\nThe first statement conveys the idea that the LLM needs to act as a specific persona and provide outputs that such a persona would. This persona can be expressed in a number of ways, ranging from a job description, title, fictional char- acter, historical figure, etc. The persona should elicit a set of attributes associated with a well-known job title, type of person, etc.2\\n\\n5) Consequences: One consideration when designing the prompt is how much to dictate to the LLM regarding what information to collect prior to termination. In the example above, the flipped interaction is open-ended and can vary sig- nificantly in the final generated artifact. This open-endedness makes the prompt generic and reusable, but may potentially ask additional questions that could be skipped if more context is given.', metadata={'source': '../2302.11382.pdf', 'start_index': 37872}),\n",
+       "  Document(page_content='In this example, the LLM is instructed to provide outputs that a ”security reviewer” would. The prompt further sets the stage that code is going to be evaluated. Finally, the user refines the persona by scoping the persona further to outputs regarding the code.\\n\\nPersonas can also represent inanimate or non-human en- tities, such as a Linux terminal, a database, or an animal’s perspective. When using this pattern to represent these entities, it can be useful to also specify how you want the inputs delivered to the entity, such as “assume my input is what the owner is saying to the dog and your output is the sounds the dog is making”. An example prompt for a non-human entity that uses a “pretend to be” wording is shown below:\\n\\n“You are going to pretend to be a Linux terminal for a computer that has been compromised by an attacker. When I type in a command, you are going the Linux to output terminal would produce.”\\n\\nthe corresponding text\\n\\nthat', metadata={'source': '../2302.11382.pdf', 'start_index': 41330}),\n",
+       "  Document(page_content='the corresponding text\\n\\nthat\\n\\nThis prompt is designed to simulate a computer that has been compromised by an attacker and is being controlled through a Linux terminal. The prompt specifies that the user will input commands into the terminal, and in response, the simulated terminal will output the corresponding text that would be produced by a real Linux terminal. This prompt is more prescriptive in the persona and asks the LLM to, not only be a Linux terminal, but to further act as a computer that has been compromised by an attacker.\\n\\n3) Structure and Key Ideas: Fundamental contextual state-\\n\\nments:\\n\\nContextual Statements Within scope X, suggest a better version of the question to use instead (Optional) prompt me if I would like to use the better version instead', metadata={'source': '../2302.11382.pdf', 'start_index': 42256})]}"
+      ]
+     },
+     "execution_count": null,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "#show result\n",
+    "ans"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "In conclusion, this is looking pretty solid. Let's leverage this functionality within the code base."
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "python3",
+   "language": "python",
+   "name": "python3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

lo-achievement/nbs/nbdev.yml

@@ -0,0 +1,9 @@
+project:
+  output-dir: _docs
+
+website:
+  title: "ai_classroom_suite"
+  site-url: "https://vanderbilt-data-science.github.io/lo-achievement"
+  description: "A repository supporting enhanced instruction and grading using AI"
+  repo-branch: main
+  repo-url: "https://github.com/vanderbilt-data-science/lo-achievement"

lo-achievement/nbs/prompt_interaction_base.ipynb

@@ -0,0 +1,482 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# prompt_interaction_base.ipynb\n",
+    "> A notebook for formulating prompts and prompting\n",
+    "\n",
+    "In this notebook, we create some base functionality for creating prompts and getting answers for the LLMs in a simplified, unified way.\n",
+    "\n",
+    ":::{.callout-caution}\n",
+    "These notebooks are development notebooks, meaning that they are meant to be run locally or somewhere that supports navigating a full repository (in other words, not Google Colab unless you clone the entire repository to drive and then mount the Drive-Repository.) However, it is expected if you're able to do all of those steps, you're likely also able to figure out the required pip installs for development there.\n",
+    ":::\n"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "skip_exec: true\n",
+    "---"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#| default_exp PromptInteractionBase"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#| export\n",
+    "from langchain.chat_models import ChatOpenAI\n",
+    "from langchain.llms import OpenAI\n",
+    "\n",
+    "from langchain import PromptTemplate\n",
+    "from langchain.prompts import ChatPromptTemplate, PromptTemplate\n",
+    "from langchain.prompts import SystemMessagePromptTemplate, HumanMessagePromptTemplate\n",
+    "from langchain.chains import LLMChain, ConversationalRetrievalChain, RetrievalQAWithSourcesChain\n",
+    "from langchain.chains.base import Chain\n",
+    "\n",
+    "from getpass import getpass\n",
+    "\n",
+    "import os"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Model and Authentication Setup\n",
+    "Here, we create functionality to authenticate the user when needed specifically using OpenAI models. Additionally, we create the capacity to make LLMChains and other chains using one unified interface."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#| export\n",
+    "def create_model(openai_mdl='gpt-3.5-turbo-16k', temperature=0.1, **chatopenai_kwargs):\n",
+    "    llm = ChatOpenAI(model_name = openai_mdl, temperature=temperature, **chatopenai_kwargs)\n",
+    "\n",
+    "    return llm"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#| export\n",
+    "def set_openai_key():\n",
+    "    openai_api_key = getpass()\n",
+    "    os.environ[\"OPENAI_API_KEY\"] = openai_api_key\n",
+    "\n",
+    "    return"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "**And now for a quick test of this functionality**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "set_openai_key()\n",
+    "assert os.environ[\"OPENAI_API_KEY\"], \"Either you didn't run set_openai_key or you haven't set it to something.\"\n",
+    "\n",
+    "chat_mdl = create_model()\n",
+    "assert isinstance(chat_mdl, ChatOpenAI), \"The default model type is currently ChatOpenAI. If that has changed, change this test.\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Create chat prompt templates\n",
+    "Here, we'll create a tutor prompt template to help us with self-study and quizzing, and help create the student messages."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#| export\n",
+    "# Create system prompt template\n",
+    "SYSTEM_TUTOR_TEMPLATE = (\"You are a world-class tutor helping students to perform better on oral and written exams though interactive experiences. \" +\n",
+    "                         \"When assessing and evaluating students, you always ask one question at a time, and wait for the student's response before \" +\n",
+    "                         \"providing them with feedback. Asking one question at a time, waiting for the student's response, and then commenting \" +\n",
+    "                         \"on the strengths and weaknesses of their responses (when appropriate) is what makes you such a sought-after, world-class tutor.\")\n",
+    "\n",
+    "# Create a human response template\n",
+    "HUMAN_RESPONSE_TEMPLATE = (\"I'm trying to better understand the text provided below. {assessment_request} The learning objectives to be assessed are: \" +\n",
+    "                           \"{learning_objectives}. Although I may request more than one assessment question, you should \" +\n",
+    "                           \"only provide ONE question in you initial response. Do not include the answer in your response. \" +\n",
+    "                           \"If I get an answer wrong, provide me with an explanation of why it was incorrect, and then give me additional \" +\n",
+    "                           \"chances to respond until I get the correct choice. Explain why the correct choice is right. \" +\n",
+    "                           \"The text that you will base your questions on is as follows: {context}.\")\n",
+    "\n",
+    "HUMAN_RETRIEVER_RESPONSE_TEMPLATE = (\"I want to master the topics based on the excerpts of the text below. Given the following extracted text from long documents, {assessment_request} The learning objectives to be assessed are: \" +\n",
+    "                           \"{learning_objectives}. Although I may request more than one assessment question, you should \" +\n",
+    "                           \"only provide ONE question in you initial response. Do not include the answer in your response. \" +\n",
+    "                           \"If I get an answer wrong, provide me with an explanation of why it was incorrect, and then give me additional \" +\n",
+    "                           \"chances to respond until I get the correct choice. Explain why the correct choice is right. \" +\n",
+    "                           \"The extracted text from long documents are as follows: {summaries}.\")\n",
+    "\n",
+    "def create_base_tutoring_prompt(system_prompt=None, human_prompt=None):\n",
+    "\n",
+    "    #setup defaults using defined values\n",
+    "    if system_prompt == None:\n",
+    "        system_prompt = PromptTemplate(template = SYSTEM_TUTOR_TEMPLATE,\n",
+    "                          input_variables = [])\n",
+    "    \n",
+    "    if human_prompt==None:\n",
+    "        human_prompt = PromptTemplate(template = HUMAN_RESPONSE_TEMPLATE,\n",
+    "                           input_variables=['assessment_request', 'learning_objectives', 'context'])\n",
+    "\n",
+    "    # Create prompt messages\n",
+    "    system_tutor_msg = SystemMessagePromptTemplate(prompt=system_prompt)\n",
+    "    human_tutor_msg = HumanMessagePromptTemplate(prompt= human_prompt)\n",
+    "\n",
+    "    # Create ChatPromptTemplate\n",
+    "    chat_prompt = ChatPromptTemplate.from_messages([system_tutor_msg, human_tutor_msg])\n",
+    "\n",
+    "    return chat_prompt"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Now for a quick unit test for testing..."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "chat_prompt = create_base_tutoring_prompt()\n",
+    "assert chat_prompt.messages[0].prompt.template == SYSTEM_TUTOR_TEMPLATE, \"Did not set up the first chat_prompt to be SystemMessage\"\n",
+    "assert chat_prompt.messages[1].prompt.template == HUMAN_RESPONSE_TEMPLATE, \"Did not set up the second element of chat_prompt to be HumanMessage\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Now, let's define a function that allows us to set up default variables in case the user chooses not to pass something in."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#| export\n",
+    "DEFAULT_ASSESSMENT_MSG = 'Please design a 5 question short answer quiz about the provided text.'\n",
+    "DEFAULT_LEARNING_OBJS_MSG = 'Identify and comprehend the important topics and underlying messages and connections within the text'\n",
+    "\n",
+    "def get_tutoring_prompt(context, chat_template=None, assessment_request = None, learning_objectives = None, **kwargs):\n",
+    "\n",
+    "    # set defaults\n",
+    "    if chat_template is None:\n",
+    "        chat_template = create_base_tutoring_prompt()\n",
+    "    else:\n",
+    "        if not all([prompt_var in chat_template.input_variables\n",
+    "                    for prompt_var in ['context', 'assessment_request', 'learning_objectives']]):\n",
+    "            raise KeyError('''It looks like you may have a custom chat_template. Either include context, assessment_request, and learning objectives\n",
+    "                           as input variables or create your own tutoring prompt.''')\n",
+    "\n",
+    "    if assessment_request is None and 'assessment_request':\n",
+    "        assessment_request = DEFAULT_ASSESSMENT_MSG\n",
+    "    \n",
+    "    if learning_objectives is None:\n",
+    "        learning_objectives = DEFAULT_LEARNING_OBJS_MSG\n",
+    "    \n",
+    "    # compose final prompt\n",
+    "    tutoring_prompt = chat_template.format_prompt(context=context,\n",
+    "                                                assessment_request = assessment_request,\n",
+    "                                                learning_objectives = learning_objectives,\n",
+    "                                                **kwargs)\n",
+    "    \n",
+    "    return tutoring_prompt\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "**Another quick unit test...**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[SystemMessage(content=\"You are a world-class tutor helping students to perform better on oral and written exams though interactive experiences.\\nWhen assessing and evaluating students, you always ask one question at a time, and wait for the student's response before providing them with feedback.\\nAsking one question at a time, waiting for the student's response, and then commenting on the strengths and weaknesses of their responses (when appropriate)\\nis what makes you such a sought-after, world-class tutor.\", additional_kwargs={}),\n",
+       " HumanMessage(content=\"I'm trying to better understand the text provided below. Please design a 5 question short answer quiz about the provided text. The learning objectives to be assessed are:\\nIdentify and comprehend the important topics and underlying messages and connections within the text. Although I may request more than one assessment question, you should\\nonly provide ONE question in you initial response. Do not include the answer in your response.\\nIf I get an answer wrong, provide me with an explanation of why it was incorrect, and then give me additional\\nchances to respond until I get the correct choice. Explain why the correct choice is right.\\nThe text that you will base your questions on is as follows: The dog was super pretty and cute.\", additional_kwargs={}, example=False)]"
+      ]
+     },
+     "execution_count": null,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# For defaults\n",
+    "res = get_tutoring_prompt('The dog was super pretty and cute').to_messages()\n",
+    "res"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Now, let's finally define how we can get the chat response from the model."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#| export\n",
+    "def get_tutoring_answer(context, tutor_mdl, chat_template=None, assessment_request=None, learning_objectives=None, return_dict=False, call_kwargs={}, input_kwargs={}):\n",
+    "    \n",
+    " # Get answer from chat\n",
+    "    \n",
+    "    # set defaults\n",
+    "    if assessment_request is None:\n",
+    "            assessment_request = DEFAULT_ASSESSMENT_MSG\n",
+    "    if learning_objectives is None:\n",
+    "        learning_objectives = DEFAULT_LEARNING_OBJS_MSG\n",
+    "    \n",
+    "    common_inputs = {'assessment_request':assessment_request, 'learning_objectives':learning_objectives}\n",
+    "    \n",
+    "    # get answer based on interaction type\n",
+    "    if isinstance(tutor_mdl, ChatOpenAI):\n",
+    "        human_ask_prompt = get_tutoring_prompt(context, chat_template, assessment_request, learning_objectives)\n",
+    "        tutor_answer = tutor_mdl(human_ask_prompt.to_messages())\n",
+    "\n",
+    "        if not return_dict:\n",
+    "            final_answer = tutor_answer.content\n",
+    "    \n",
+    "    elif isinstance(tutor_mdl, Chain):\n",
+    "        if isinstance(tutor_mdl, RetrievalQAWithSourcesChain):\n",
+    "            if 'question' not in input_kwargs.keys():\n",
+    "                common_inputs['question'] = assessment_request\n",
+    "            final_inputs = {**common_inputs, **input_kwargs}\n",
+    "        else:\n",
+    "            common_inputs['context'] = context\n",
+    "            final_inputs = {**common_inputs, **input_kwargs}\n",
+    "         \n",
+    "        # get answer\n",
+    "        tutor_answer = tutor_mdl(final_inputs, **call_kwargs)\n",
+    "        final_answer = tutor_answer\n",
+    "\n",
+    "        if not return_dict:\n",
+    "            final_answer = final_answer['answer']\n",
+    "    \n",
+    "    else:\n",
+    "        raise NotImplementedError(f\"tutor_mdl of type {type(tutor_mdl)} is not supported.\")\n",
+    "\n",
+    "    return final_answer"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#| export\n",
+    "\n",
+    "DEFAULT_CONDENSE_PROMPT_TEMPLATE = (\"Given the following conversation and a follow up question, rephrase the follow up question to be a standalone question, \" + \n",
+    "                                    \"in its original language.\\n\\nChat History:\\n{chat_history}\\nFollow Up Input: {question}\\nStandalone question:\")\n",
+    "\n",
+    "DEFAULT_QUESTION_PROMPT_TEMPLATE  = (\"Use the following portion of a long document to see if any of the text is relevant to creating a response to the question.\" +\n",
+    "                                     \"\\nReturn any relevant text verbatim.\\n{context}\\nQuestion: {question}\\nRelevant text, if any:\")\n",
+    "\n",
+    "DEFAULT_COMBINE_PROMPT_TEMPLATE = (\"Given the following extracted parts of a long document and the given prompt, create a final answer with references ('SOURCES'). \"+\n",
+    "                                   \"If you don't have a response, just say that you are unable to come up with a response. \"+\n",
+    "                                   \"\\nSOURCES:\\n\\nQUESTION: {question}\\n=========\\n{summaries}\\n=========\\nFINAL ANSWER:'\")\n",
+    "\n",
+    "def create_tutor_mdl_chain(kind='llm', mdl=None, prompt_template = None, **kwargs):\n",
+    "    \n",
+    "    #Validate parameters\n",
+    "    if mdl is None:\n",
+    "        mdl = create_model()\n",
+    "    kind = kind.lower()\n",
+    "    \n",
+    "    #Create model chain\n",
+    "    if kind == 'llm':\n",
+    "        if prompt_template is None:\n",
+    "            prompt_template = create_base_tutoring_prompt()\n",
+    "        mdl_chain = LLMChain(llm=mdl, prompt=prompt_template, **kwargs)\n",
+    "    elif kind == 'conversational':\n",
+    "        if prompt_template is None:\n",
+    "            prompt_template = PromptTemplate.from_template(DEFAULT_CONDENSE_PROMPT_TEMPLATE)\n",
+    "        mdl_chain = ConversationalRetrieverChain.from_llm(mdl, condense_question_prompt = prompt_template, **kwargs)\n",
+    "    elif kind == 'retrieval_qa':\n",
+    "        if prompt_template is None:\n",
+    "\n",
+    "            #Create custom human prompt to take in summaries\n",
+    "            human_prompt = PromptTemplate(template = HUMAN_RETRIEVER_RESPONSE_TEMPLATE,\n",
+    "                           input_variables=['assessment_request', 'learning_objectives', 'summaries'])\n",
+    "            prompt_template = create_base_tutoring_prompt(human_prompt=human_prompt)\n",
+    "            \n",
+    "        #Create the combination prompt and model\n",
+    "        question_template = PromptTemplate.from_template(DEFAULT_QUESTION_PROMPT_TEMPLATE)\n",
+    "        mdl_chain = RetrievalQAWithSourcesChain.from_llm(llm=mdl, question_prompt=question_template, combine_prompt = prompt_template, **kwargs)\n",
+    "    else:\n",
+    "        raise NotImplementedError(f\"Model kind {kind} not implemented\")\n",
+    "    \n",
+    "    return mdl_chain"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "**Another brief test of behavior of these functions**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res = get_tutoring_answer('The dog is super cute', chat_mdl)\n",
+    "print(res)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Validate LLM Chain"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Try llm model chain, making sure we've set the API key\n",
+    "llm_chain_test = create_tutor_mdl_chain('llm')\n",
+    "res = llm_chain_test.run({'context':'some context', 'assessment_request':'some assessment', 'learning_objectives':'some prompt'})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "<class 'langchain.chains.llm.LLMChain'>\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "'Sure, I can help you with that. Please provide me with the specific text that you would like me to base my questions on.'"
+      ]
+     },
+     "execution_count": null,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Verify information about the cell above\n",
+    "print(type(llm_chain_test))\n",
+    "print(res)\n",
+    "\n",
+    "# unit tests\n",
+    "assert isinstance(llm_chain_test, LLMChain), 'the output of llm create_tutor_mdl_chain should be an instance of LLMChain'\n",
+    "assert isinstance(res, str), 'the output of running the llm chain should be of type string.'"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Now, we'll try this with just the default function to run things..."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'context': 'some context',\n",
+       " 'assessment_request': 'Please design a 5 question short answer quiz about the provided text.',\n",
+       " 'learning_objectives': 'Identify and comprehend the important topics and underlying messages and connections within the text',\n",
+       " 'text': 'Question 1: What are the main topics discussed in the text?\\n\\n(Note: Please provide your answer and I will provide feedback accordingly.)'}"
+      ]
+     },
+     "execution_count": null,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "res = get_tutoring_answer(context='some context', tutor_mdl = llm_chain_test)\n",
+    "res"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "OK, this base functionality is looking good."
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "python3",
+   "language": "python",
+   "name": "python3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

lo-achievement/nbs/self_study_prompts.ipynb

@@ -0,0 +1,342 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# self_study_prompts.ipynb\n",
+    "> A listing of all prompts for self-study\n",
+    "\n",
+    "This notebook contains all prompts used for self-study as a central place that can be monitored and evaluated for appropriate functionality. Note that these perform the requests part of the prompts.\n",
+    "\n",
+    ":::{.callout-caution}\n",
+    "These notebooks are development notebooks, meaning that they are meant to be run locally or somewhere that supports navigating a full repository (in other words, not Google Colab unless you clone the entire repository to drive and then mount the Drive-Repository.) However, it is expected if you're able to do all of those steps, you're likely also able to figure out the required pip installs for development there.\n",
+    ":::"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "skip_exec: true\n",
+    "---"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#| default_exp SelfStudyPrompts"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Self-study texts\n",
+    "We'll now define the text for our self-study questions. Note that these will align with `assessment_request` in the `PromptInteractionBase` module."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#| export\n",
+    "# used for pretty display\n",
+    "import pandas as pd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#| export\n",
+    "MC_QUIZ_DEFAULT = \"Please design a 5 question multiple choice quiz about the provided text.\"\n",
+    "\n",
+    "SHORT_ANSWER_DEFAULT = (\"Please design a 5 question short answer quiz about the provided text. \"\n",
+    "                        \"The question types should be short answer. Expect the correct answers to be a few sentences long.\")\n",
+    "\n",
+    "FILL_BLANK_DEFAULT = \"\"\"Create a 5 question fill in the blank quiz referencing parts of the provided text.\n",
+    "The \"blank\" part of the question should appear as \"________\". The answers should reflect what word(s) should go in the blank an accurate statement.\n",
+    "An example is as follows: \"The author of the book is ______.\" The question should be a statement.\n",
+    "\"\"\"\n",
+    "\n",
+    "SEQUENCING_DEFAULT = \"\"\"Create a 5 question questionnaire that will ask me to recall the steps or sequence of events\n",
+    "in the provided text.\"\"\"\n",
+    "\n",
+    "RELATIONSHIP_DEFAULT = (\"Create a 5 question quiz for the student that asks the student to identify relationships between\"\n",
+    "                        \"topics or concepts that are important to understanding this text.\")\n",
+    "\n",
+    "CONCEPTS_DEFAULT = \"\"\" Design a 5 question quiz that asks me about definitions or concepts of importance in the provided text.\"\"\"\n",
+    "\n",
+    "REAL_WORLD_EXAMPLE_DEFAULT = \"\"\"Demonstrate how the provided context can be applied to solve a real world problem.\n",
+    "Ask me questions about how the demonstration you provided relates to solving a real world problem.\"\"\"\n",
+    "\n",
+    "RANDOMIZED_QUESTIONS_DEFAULT = \"\"\"Generate a high-quality assessment consisting of 5 varied questions,\n",
+    "each of different types (open-ended, multiple choice, short answer, analogies, etc.)\"\"\"\n",
+    "\n",
+    "SELF_STUDY_PROMPT_NAMES = ['MC_QUIZ_DEFAULT',\n",
+    "'SHORT_ANSWER_DEFAULT',\n",
+    "'FILL_BLANK_DEFAULT',\n",
+    "'SEQUENCING_DEFAULT',\n",
+    "'RELATIONSHIP_DEFAULT',\n",
+    "'CONCEPTS_DEFAULT',\n",
+    "'REAL_WORLD_EXAMPLE_DEFAULT',\n",
+    "'RANDOMIZED_QUESTIONS_DEFAULT']"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Create functions to assist with creating prompts\n",
+    "Now, we'll use this section in order to create some functions which will allow the user to display all available prompts."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#| export\n",
+    "# Define self study dictionary for lookup\n",
+    "SELF_STUDY_DEFAULTS = {'mc': MC_QUIZ_DEFAULT,\n",
+    "'short_answer': SHORT_ANSWER_DEFAULT,\n",
+    "'fill_blank': FILL_BLANK_DEFAULT,\n",
+    "'sequencing': SEQUENCING_DEFAULT,\n",
+    "'relationships': RELATIONSHIP_DEFAULT,\n",
+    "'concepts': CONCEPTS_DEFAULT,\n",
+    "'real_world_example': REAL_WORLD_EXAMPLE_DEFAULT,\n",
+    "'randomized_questions': RANDOMIZED_QUESTIONS_DEFAULT\n",
+    "} \n",
+    "\n",
+    "# Return list of all self study prompts\n",
+    "def list_all_self_study_prompt_keys():\n",
+    "    return list(SELF_STUDY_DEFAULTS.keys())\n",
+    "\n",
+    "def list_all_self_study_prompts():\n",
+    "    return list(SELF_STUDY_DEFAULTS.values())\n",
+    "    \n",
+    "# Return list of all self study variable names\n",
+    "def list_default_self_prompt_varnames():\n",
+    "    return SELF_STUDY_PROMPT_NAMES\n",
+    "\n",
+    "# Print as a table\n",
+    "def print_all_self_study_prompts():\n",
+    "    with pd.option_context('max_colwidth', None):\n",
+    "        display(pd.DataFrame({'SELF_STUDY_DEFAULTS key': list(SELF_STUDY_DEFAULTS.keys()),\n",
+    "        'Prompt': list(SELF_STUDY_DEFAULTS.values())}))\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Now, we'll have quick unit test just to make sure this is working correctly."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['mc',\n",
+       " 'short_answers',\n",
+       " 'fill_blanks',\n",
+       " 'sequencing',\n",
+       " 'relationships',\n",
+       " 'concepts',\n",
+       " 'real_world_example',\n",
+       " 'randomized_questions']"
+      ]
+     },
+     "execution_count": null,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "list_all_self_study_prompt_keys()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['Please design a 5 question quiz about the provided text.',\n",
+       " 'Please design a 5 question short answer quiz about the provided text. The question types should be short answer. Expect the correct answers to be a few sentences long.',\n",
+       " 'Create a 5 question fill in the blank quiz referencing parts of the provided text.\\nThe \"blank\" part of the question should appear as \"________\". The answers should reflect what word(s) should go in the blank an accurate statement.\\nAn example is as follows: \"The author of the book is ______.\" The question should be a statement.\\n',\n",
+       " 'Create a 5 question questionnaire that will ask me to recall the steps or sequence of events\\nin the provided text.',\n",
+       " 'Please design a 5 question quiz that asks me to draw or explain relationships\\nbetween important concepts or topics in the provided text.',\n",
+       " ' Design a 5 question quiz that asks me about definitions or concepts of importance in the provided text.',\n",
+       " 'Demonstrate how the provided context can be applied to solve a real world problem.\\nAsk me questions about how the demonstration you provided relates to solving a real world problem.',\n",
+       " 'Generate a high-quality assessment consisting of 5 varied questions,\\neach of different types (open-ended, multiple choice, short answer, analogies, etc.)']"
+      ]
+     },
+     "execution_count": null,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "list_all_self_study_prompts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['MC_QUIZ_DEFAULT',\n",
+       " 'SHORT_ANSWER_DEFAULT',\n",
+       " 'FILL_BLANK_DEFAULT',\n",
+       " 'SEQUENCING_DEFAULT',\n",
+       " 'RELATIONSHIP_DEFAULT',\n",
+       " 'CONCEPTS_DEFAULT',\n",
+       " 'REAL_WORLD_EXAMPLE_DEFAULT',\n",
+       " 'RANDOMIZED_QUESTIONS_DEFAULT']"
+      ]
+     },
+     "execution_count": null,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "list_default_self_prompt_varnames()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Variable Name</th>\n",
+       "      <th>Prompt</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>mc</td>\n",
+       "      <td>Please design a 5 question quiz about the provided text.</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>short_answers</td>\n",
+       "      <td>Please design a 5 question short answer quiz about the provided text. The question types should be short answer. Expect the correct answers to be a few sentences long.</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>fill_blanks</td>\n",
+       "      <td>Create a 5 question fill in the blank quiz referencing parts of the provided text.\\nThe \"blank\" part of the question should appear as \"________\". The answers should reflect what word(s) should go in the blank an accurate statement.\\nAn example is as follows: \"The author of the book is ______.\" The question should be a statement.\\n</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>sequencing</td>\n",
+       "      <td>Create a 5 question questionnaire that will ask me to recall the steps or sequence of events\\nin the provided text.</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>relationships</td>\n",
+       "      <td>Please design a 5 question quiz that asks me to draw or explain relationships\\nbetween important concepts or topics in the provided text.</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>concepts</td>\n",
+       "      <td>Design a 5 question quiz that asks me about definitions or concepts of importance in the provided text.</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6</th>\n",
+       "      <td>real_world_example</td>\n",
+       "      <td>Demonstrate how the provided context can be applied to solve a real world problem.\\nAsk me questions about how the demonstration you provided relates to solving a real world problem.</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>7</th>\n",
+       "      <td>randomized_questions</td>\n",
+       "      <td>Generate a high-quality assessment consisting of 5 varied questions,\\neach of different types (open-ended, multiple choice, short answer, analogies, etc.)</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "          Variable Name  \\\n",
+       "0                    mc   \n",
+       "1         short_answers   \n",
+       "2           fill_blanks   \n",
+       "3            sequencing   \n",
+       "4         relationships   \n",
+       "5              concepts   \n",
+       "6    real_world_example   \n",
+       "7  randomized_questions   \n",
+       "\n",
+       "                                                                                                                                                                                                                                                                                                                                         Prompt  \n",
+       "0                                                                                                                                                                                                                                                                                      Please design a 5 question quiz about the provided text.  \n",
+       "1                                                                                                                                                                       Please design a 5 question short answer quiz about the provided text. The question types should be short answer. Expect the correct answers to be a few sentences long.  \n",
+       "2  Create a 5 question fill in the blank quiz referencing parts of the provided text.\\nThe \"blank\" part of the question should appear as \"________\". The answers should reflect what word(s) should go in the blank an accurate statement.\\nAn example is as follows: \"The author of the book is ______.\" The question should be a statement.\\n  \n",
+       "3                                                                                                                                                                                                                           Create a 5 question questionnaire that will ask me to recall the steps or sequence of events\\nin the provided text.  \n",
+       "4                                                                                                                                                                                                     Please design a 5 question quiz that asks me to draw or explain relationships\\nbetween important concepts or topics in the provided text.  \n",
+       "5                                                                                                                                                                                                                                       Design a 5 question quiz that asks me about definitions or concepts of importance in the provided text.  \n",
+       "6                                                                                                                                                        Demonstrate how the provided context can be applied to solve a real world problem.\\nAsk me questions about how the demonstration you provided relates to solving a real world problem.  \n",
+       "7                                                                                                                                                                                    Generate a high-quality assessment consisting of 5 varied questions,\\neach of different types (open-ended, multiple choice, short answer, analogies, etc.)  "
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "print_all_self_study_prompts()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "python3",
+   "language": "python",
+   "name": "python3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

lo-achievement/nbs/styles.css

@@ -0,0 +1,37 @@
+.cell {
+  margin-bottom: 1rem;
+}
+
+.cell > .sourceCode {
+  margin-bottom: 0;
+}
+
+.cell-output > pre {
+  margin-bottom: 0;
+}
+
+.cell-output > pre, .cell-output > .sourceCode > pre, .cell-output-stdout > pre {
+  margin-left: 0.8rem;
+  margin-top: 0;
+  background: none;
+  border-left: 2px solid lightsalmon;
+  border-top-left-radius: 0;
+  border-top-right-radius: 0;
+}
+
+.cell-output > .sourceCode {
+  border: none;
+}
+
+.cell-output > .sourceCode {
+  background: none;
+  margin-top: 0;
+}
+
+div.description {
+  padding-left: 2px;
+  padding-top: 5px;
+  font-style: italic;
+  font-size: 135%;
+  opacity: 70%;
+}

lo-achievement/prompt_with_context.ipynb

@@ -0,0 +1,796 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "<a href=\"https://colab.research.google.com/github/vanderbilt-data-science/lo-achievement/blob/main/prompt_with_context.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# LLMs for Self-Study\n",
+    "> A prompt and code template for better understanding texts\n",
+    "\n",
+    "This notebook provides a guide for using LLMs for self-study programmatically. A number of prompt templates are provided to assist with generating great assessments for self-study, and code is additionally provided for fast usage. This notebook is best leveraged for a set of documents (text or PDF preferred) **to be uploaded** for interaction with the model.\n",
+    "\n",
+    "This version of the notebook is best suited for those who prefer to use files from their local drive as context rather than copy and pasting directly into the notebook to be used as context for the model. If you prefer to copy and paste text, you should direct yourself to the [prompt_with_context](https://colab.research.google.com/github/vanderbilt-data-science/lo-achievement/blob/main/prompt_with_context.ipynb) notebook."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Code Setup\n",
+    "Run the following cells to setup the rest of the environment for prompting. In the following section, we set up the computational environment with imported code, setup your API key access to OpenAI, and loading access to your language model. Note that the following cells may take a long time to run."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Library installation and loading\n",
+    "The following `pip install` code should be run if you're using Google Colab, or otherwise do not have a computational environment (e.g., _venv_, _conda virtual environment_, _Docker, Singularity, or other container_) with these packages installed."
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "skip_exec: true\n",
+    "---"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# run this code if you're using Google Colab or don't have these packages installed in your computing environment\n",
+    "! pip install pip install git+https://<token>@github.com/vanderbilt-data-science/lo-achievement.git"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# import required libraries\n",
+    "import numpy as np\n",
+    "import getpass\n",
+    "import os\n",
+    "from langchain.chat_models import ChatOpenAI\n",
+    "from langchain.chains import RetrievalQA\n",
+    "from langchain.schema import SystemMessage, HumanMessage, AIMessage"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# libraries from our package\n",
+    "from ai_classroom_suite.PromptInteractionBase import *\n",
+    "import ai_classroom_suite.SelfStudyPrompts as ssp"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## API and model setup\n",
+    "\n",
+    "Use these cells to load the API keys required for this notebook and create a basic OpenAI LLM model. The code below uses the variable you created above when you input your API Key."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Set up OpenAI API Key\n",
+    "set_openai_key()\n",
+    "\n",
+    "# Create model\n",
+    "mdl = create_model()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Inspect Available Default Prompts\n",
+    "A number of default prompts have been provided for you so you don't need to form your own prompt to begin with. They will be listed below, and the different ways to interact with them are displayed."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>SELF_STUDY_DEFAULTS key</th>\n",
+       "      <th>Prompt</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>mc</td>\n",
+       "      <td>Please design a 5 question multiple choice quiz about the provided text.</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>short_answer</td>\n",
+       "      <td>Please design a 5 question short answer quiz about the provided text. The question types should be short answer. Expect the correct answers to be a few sentences long.</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>fill_blank</td>\n",
+       "      <td>Create a 5 question fill in the blank quiz referencing parts of the provided text.\\nThe \"blank\" part of the question should appear as \"________\". The answers should reflect what word(s) should go in the blank an accurate statement.\\nAn example is as follows: \"The author of the book is ______.\" The question should be a statement.\\n</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>sequencing</td>\n",
+       "      <td>Create a 5 question questionnaire that will ask me to recall the steps or sequence of events\\nin the provided text.</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>relationships</td>\n",
+       "      <td>Create a 5 question quiz for the student that asks the student to identify relationships betweentopics or concepts that are important to understanding this text.</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>concepts</td>\n",
+       "      <td>Design a 5 question quiz that asks me about definitions or concepts of importance in the provided text.</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6</th>\n",
+       "      <td>real_world_example</td>\n",
+       "      <td>Demonstrate how the provided context can be applied to solve a real world problem.\\nAsk me questions about how the demonstration you provided relates to solving a real world problem.</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>7</th>\n",
+       "      <td>randomized_questions</td>\n",
+       "      <td>Generate a high-quality assessment consisting of 5 varied questions,\\neach of different types (open-ended, multiple choice, short answer, analogies, etc.)</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "  SELF_STUDY_DEFAULTS key  \\\n",
+       "0                      mc   \n",
+       "1            short_answer   \n",
+       "2              fill_blank   \n",
+       "3              sequencing   \n",
+       "4           relationships   \n",
+       "5                concepts   \n",
+       "6      real_world_example   \n",
+       "7    randomized_questions   \n",
+       "\n",
+       "                                                                                                                                                                                                                                                                                                                                         Prompt  \n",
+       "0                                                                                                                                                                                                                                                                      Please design a 5 question multiple choice quiz about the provided text.  \n",
+       "1                                                                                                                                                                       Please design a 5 question short answer quiz about the provided text. The question types should be short answer. Expect the correct answers to be a few sentences long.  \n",
+       "2  Create a 5 question fill in the blank quiz referencing parts of the provided text.\\nThe \"blank\" part of the question should appear as \"________\". The answers should reflect what word(s) should go in the blank an accurate statement.\\nAn example is as follows: \"The author of the book is ______.\" The question should be a statement.\\n  \n",
+       "3                                                                                                                                                                                                                           Create a 5 question questionnaire that will ask me to recall the steps or sequence of events\\nin the provided text.  \n",
+       "4                                                                                                                                                                             Create a 5 question quiz for the student that asks the student to identify relationships betweentopics or concepts that are important to understanding this text.  \n",
+       "5                                                                                                                                                                                                                                       Design a 5 question quiz that asks me about definitions or concepts of importance in the provided text.  \n",
+       "6                                                                                                                                                        Demonstrate how the provided context can be applied to solve a real world problem.\\nAsk me questions about how the demonstration you provided relates to solving a real world problem.  \n",
+       "7                                                                                                                                                                                    Generate a high-quality assessment consisting of 5 varied questions,\\neach of different types (open-ended, multiple choice, short answer, analogies, etc.)  "
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "# show all prompts and names\n",
+    "ssp.print_all_self_study_prompts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'Please design a 5 question multiple choice quiz about the provided text.'"
+      ]
+     },
+     "execution_count": null,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# accessing texts of desired assessment types\n",
+    "ssp.SELF_STUDY_DEFAULTS['mc']"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Add your context and assign the prefix to your query.\n",
+    "The query assigned here serves as an example."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "context = \"\"\" Two roads diverged in a yellow wood,\n",
+    "And sorry I could not travel both\n",
+    "And be one traveler, long I stood\n",
+    "And looked down one as far as I could\n",
+    "To where it bent in the undergrowth;\n",
+    "Then took the other, as just as fair,\n",
+    "And having perhaps the better claim,\n",
+    "Because it was grassy and wanted wear;\n",
+    "Though as for that the passing there\n",
+    "Had worn them really about the same,\n",
+    "And both that morning equally lay\n",
+    "In leaves no step had trodden black.\n",
+    "Oh, I kept the first for another day!\n",
+    "Yet knowing how way leads on to way,\n",
+    "I doubted if I should ever come back.\n",
+    "I shall be telling this with a sigh\n",
+    "Somewhere ages and ages hence:\n",
+    "Two roads diverged in a wood, and I—\n",
+    "I took the one less traveled by,\n",
+    "And that has made all the difference.\n",
+    "—-Robert Frost—-\n",
+    "Education Place: http://www.eduplace.com \"\"\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# A guide to prompting for self-study\n",
+    "In this section, we provide a number of different approaches for using AI to help you assess and explain the knowledge of your document. Start by interacting with the model and then try out the rest of the prompts!"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Interact with the model\n",
+    "\n",
+    "Now that your context is created, you can begin interacting with the model! Below, we have a comprehensive list of examples using different question types, but feel free to use this code block to experiment with the model.\n",
+    "\n",
+    "First, let's make the settings for the query. In other words, what are the learning objectives and what is the type of assessment we want to have?"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# set short answer as the assessment type\n",
+    "assessment_type = ssp.SELF_STUDY_DEFAULTS[\"short_answer\"]\n",
+    "\n",
+    "# set learning objectives if desired\n",
+    "learning_objs = (\"\"\"1. Identify the key elements of the poem: narrator, setting, and underlying message.\n",
+    "                 2. Understand the literary devices used in poetry and their purposes.\"\"\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Next, let's use the predefined defaults with the model and provided APIs from `ai_classroom_suite`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'Question 1: Who is the narrator of the poem and what is the setting?\\n\\nPlease provide your answer in a few sentences.'"
+      ]
+     },
+     "execution_count": null,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Ask the tutor to prompt you based on the text\n",
+    "get_tutoring_answer(context, mdl, assessment_request = assessment_type,\n",
+    "                    learning_objectives = learning_objs)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "The complete prompt sent is based on pre-generated information and can be seen below. Your context, assessment_type, and learning objectives are substituted to create a full prompt as shown below."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "System: You are a world-class tutor helping students to perform better on oral and written exams though interactive experiences. When assessing and evaluating students, you always ask one question at a time, and wait for the student's response before providing them with feedback. Asking one question at a time, waiting for the student's response, and then commenting on the strengths and weaknesses of their responses (when appropriate) is what makes you such a sought-after, world-class tutor.\n",
+      "Human: I'm trying to better understand the text provided below. Please design a 5 question short answer quiz about the provided text. The question types should be short answer. Expect the correct answers to be a few sentences long. The learning objectives to be assessed are: 1. Identify the key elements of the poem: narrator, setting, and underlying message.\n",
+      "                 2. Understand the literary devices used in poetry and their purposes.. Although I may request more than one assessment question, you should only provide ONE question in you initial response. Do not include the answer in your response. If I get an answer wrong, provide me with an explanation of why it was incorrect, and then give me additional chances to respond until I get the correct choice. Explain why the correct choice is right. The text that you will base your questions on is as follows:  Two roads diverged in a yellow wood,\n",
+      "And sorry I could not travel both\n",
+      "And be one traveler, long I stood\n",
+      "And looked down one as far as I could\n",
+      "To where it bent in the undergrowth;\n",
+      "Then took the other, as just as fair,\n",
+      "And having perhaps the better claim,\n",
+      "Because it was grassy and wanted wear;\n",
+      "Though as for that the passing there\n",
+      "Had worn them really about the same,\n",
+      "And both that morning equally lay\n",
+      "In leaves no step had trodden black.\n",
+      "Oh, I kept the first for another day!\n",
+      "Yet knowing how way leads on to way,\n",
+      "I doubted if I should ever come back.\n",
+      "I shall be telling this with a sigh\n",
+      "Somewhere ages and ages hence:\n",
+      "Two roads diverged in a wood, and I—\n",
+      "I took the one less traveled by,\n",
+      "And that has made all the difference.\n",
+      "—-Robert Frost—-\n",
+      "Education Place: http://www.eduplace.com .\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Use different function to create the prompt\n",
+    "full_prompt = get_tutoring_prompt(context, assessment_request = assessment_type,\n",
+    "                                  learning_objectives = learning_objs)\n",
+    "\n",
+    "# Show the prompt as a string\n",
+    "print(full_prompt.to_string())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Alternately, you can define your own prompt, which you'll as appropriate. To modify the kind of assessment you'll be asking for, change `assessment_request`. An example of how to add more context to the model is shown below as well."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "\"Question: Who is the narrator in the poem and what is the underlying message conveyed?\\n\\nHint: Pay attention to the pronouns used throughout the poem to determine the narrator's identity. Additionally, think about the choices made by the narrator and the impact those choices have on their life. The underlying message is related to the consequences of these choices.\\n\\nTake your time to reflect on the text and provide your answer when you're ready.\""
+      ]
+     },
+     "execution_count": null,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Use your own texts\n",
+    "custom_request = (\"Ask me a short answer question about the provided text. The questions you ask should allow\"\n",
+    "                  \" me to demonstrate my creativity, capacity for out-of-the-box thinking, insights, and deeper meaning \"\n",
+    "                  \" of the text.\")\n",
+    "additional_context = (\"This is a text written by Robert Frost, a famous American poet. The text is widely studied in K-12 literature\"\n",
+    "                    \" education courses, and should be read with an eye towards the philosophical and human themes of the text.\")\n",
+    "\n",
+    "# Concatenate context\n",
+    "informed_context = context + \"\\n Additional information about the text is: \" + additional_context \n",
+    "\n",
+    "# Use custom_request defined as the assessment request\n",
+    "get_tutoring_answer(informed_context, mdl, assessment_request = custom_request,\n",
+    "                    learning_objectives = learning_objs)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Types of Questions and Prompts\n",
+    "\n",
+    "Below is a comprehensive list of question types and prompt templates designed by our team. There are also example code blocks, where you can see how the model performed with the example and try it for yourself using the prompt template."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Multiple Choice\n",
+    "\n",
+    "Prompt: The following text should be used as the basis for the instructions which follow: {context}. Please design a {number of questions} question quiz about {name or reference to context} which reflects the learning objectives: {list of learning objectives}. The questions should be multiple choice. Provide one question at a time, and wait for my response before providing me with feedback. Again, while the quiz may ask for multiple questions, you should only provide ONE question in you initial response. Do not include the answer in your response. If I get an answer wrong, provide me with an explanation of why it was incorrect,and then give me additional chances to respond until I get the correct choice. Explain why the correct choice is right."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Question 1: Who is the narrator of the poem?\n",
+      "\n",
+      "A) Robert Frost\n",
+      "B) The traveler \n",
+      "C) The undergrowth \n",
+      "D) The wood\n",
+      "\n",
+      "Please provide your answer.\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Multiple choice code example\n",
+    "tutor_q = get_tutoring_answer(context, mdl, assessment_request = ssp.SELF_STUDY_DEFAULTS['mc'],\n",
+    "                              learning_objectives = learning_objs)\n",
+    "print(tutor_q)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Short Answer\n",
+    "\n",
+    "Prompt: Please design a {number of questions} question quiz about {context} which reflects the learning objectives: {list of learning objectives}. The questions should be short answer. Expect the correct answers to be {anticipated length} long. Provide one question at a time, and wait for my response before providing me with feedback. Again, while the quiz may ask for multiple questions, you should only provide ONE question in you initial response. Do not include the answer in your response. If I get an answer wrong, provide me with an explanation of why it was incorrect,and then give me additional chances to respond until I get the correct choice. Explain why the correct choice is right."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Question 1: Who is the narrator of the poem and what is the setting?\n",
+      "\n",
+      "Remember to answer the question by identifying the narrator of the poem and describing the setting in which the events take place.\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Short answer code example\n",
+    "tutor_q = get_tutoring_answer(context, mdl, assessment_request = ssp.SELF_STUDY_DEFAULTS['short_answer'],\n",
+    "                              learning_objectives = learning_objs)\n",
+    "print(tutor_q)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Fill-in-the-blank\n",
+    "\n",
+    "Prompt: Create a {number of questions} question fill in the blank quiz refrencing {context}. The quiz should reflect the learning objectives: {learning objectives}. The \"blank\" part of the question should appear as \"________\". The answers should reflect what word(s) should go in the blank an accurate statement.\n",
+    "\n",
+    "An example is the follow: \"The author of the book is \"________.\"\n",
+    "\n",
+    "The question should be a statement. Provide one question at a time, and wait for my response before providing me with feedback. Again, while the quiz may ask for multiple questions, you should only provide ONE question in you initial response. Do not include the answer in your response. If I get an answer wrong, provide me with an explanation of why it was incorrect,and then give me additional chances to respond until I get the correct choice. Explain why the correct choice is right."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Question 1: The poem \"The Road Not Taken\" was written by ________.\n",
+      "\n",
+      "Question 2: What is the color of the wood where the two roads diverged? \n",
+      "\n",
+      "Question 3: What is the reason the narrator gives for choosing the second road?\n",
+      "\n",
+      "Question 4: What does the narrator say about the wear of both roads?\n",
+      "\n",
+      "Question 5: According to the poem, what has made all the difference in the narrator's life?\n",
+      "\n",
+      "Remember to wait for the student's response before providing feedback.\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Fill in the blank code example\n",
+    "tutor_q = get_tutoring_answer(context, mdl, assessment_request = ssp.SELF_STUDY_DEFAULTS['fill_blank'],\n",
+    "                              learning_objectives = learning_objs)\n",
+    "print(tutor_q)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Sequencing\n",
+    "\n",
+    "Prompt: Please develop a {number of questions} question questionnaire that will ask me to recall the steps involved in the following learning objectives in regard to {context}: {learning objectives}. Provide one question at a time, and wait for my response before providing me with feedback. Again, while the quiz may ask for multiple questions, you should only provide ONE question in you initial response. Do not include the answer in your response. If I get an answer wrong, provide me with an explanation of why it was incorrect, and then give me additional chances to respond until I get the correct choice. After I respond, explain their sequence to me."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Question 1: Who is the narrator of the poem?\n",
+      "\n",
+      "Question 2: What is the setting of the poem?\n",
+      "\n",
+      "Question 3: What is the underlying message of the poem?\n",
+      "\n",
+      "Question 4: What literary device is used in the line \"Two roads diverged in a yellow wood\"?\n",
+      "\n",
+      "Question 5: What is the purpose of using the literary device in question 4?\n",
+      "\n",
+      "Please answer question 1 first.\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Sequence example\n",
+    "tutor_q = get_tutoring_answer(context, mdl, assessment_request = ssp.SELF_STUDY_DEFAULTS['sequencing'],\n",
+    "                              learning_objectives = learning_objs)\n",
+    "\n",
+    "print(tutor_q)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Relationships/drawing connections\n",
+    "\n",
+    "Prompt: Please design a {number of questions} question quiz that asks me to explain the relationships that exist within the following learning objectives, referencing {context}: {learning objectives}. Provide one question at a time, and wait for my response before providing me with feedback. Again, while the quiz may ask for multiple questions, you should only provide ONE question in you initial response. Do not include the answer in your response. If I get an answer wrong, provide me with an explanation of why it was incorrect,and then give me additional chances to respond until I get the correct choice. Explain why the correct choice is right."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Question 1: Who is the narrator of the poem?\n",
+      "\n",
+      "Question 2: What is the setting of the poem?\n",
+      "\n",
+      "Question 3: What is the underlying message of the poem?\n",
+      "\n",
+      "Question 4: What literary device is used when the narrator says, \"Two roads diverged in a yellow wood\"?\n",
+      "\n",
+      "Question 5: What literary device is used when the narrator says, \"I took the one less traveled by, And that has made all the difference\"?\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Relationships example\n",
+    "tutor_q = get_tutoring_answer(context, mdl, assessment_request = ssp.SELF_STUDY_DEFAULTS['relationships'],\n",
+    "                              learning_objectives = learning_objs)\n",
+    "\n",
+    "print(tutor_q)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Concepts and Definitions\n",
+    "\n",
+    "Prompt: Design a {number of questions} question quiz that asks me about definitions related to the following learning objectives: {learning objectives} - based on {context}\".\n",
+    "Provide one question at a time, and wait for my response before providing me with feedback. Again, while the quiz may ask for multiple questions, you should only provide ONE question in you initial response. Do not include the answer in your response. If I get an answer wrong, provide me with an explanation of why it was incorrect,and then give me additional chances to respond until I get the correct choice. Explain why the correct choice is right.\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Question 1: Who is the narrator of the poem? \n",
+      "\n",
+      "Remember, the narrator is the person who is speaking or telling the story.\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Concepts and definitions example\n",
+    "tutor_q = get_tutoring_answer(context, mdl, assessment_request = ssp.SELF_STUDY_DEFAULTS['concepts'],\n",
+    "                              learning_objectives = learning_objs)\n",
+    "\n",
+    "print(tutor_q)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Real Word Examples\n",
+    "\n",
+    "Prompt: Demonstrate how {context} can be applied to solve a real-world problem related to the following learning objectives: {learning objectives}. Ask me questions regarding this theory/concept.\n",
+    "\n",
+    "Provide one question at a time, and wait for my response before providing me with feedback. Again, while the quiz may ask for multiple questions, you should only provide ONE question in you initial response. Do not include the answer in your response. If I get an answer wrong, provide me with an explanation of why it was incorrect,and then give me additional chances to respond until I get the correct choice. Explain why the correct choice is right."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Question 1: Who is the narrator of the poem?\n",
+      "\n",
+      "Question 2: What is the setting of the poem?\n",
+      "\n",
+      "Question 3: What is the underlying message of the poem?\n",
+      "\n",
+      "Remember to provide your answer to one question at a time.\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Real word example\n",
+    "tutor_q = get_tutoring_answer(context, mdl, assessment_request = ssp.SELF_STUDY_DEFAULTS['real_world_example'],\n",
+    "                              learning_objectives = learning_objs)\n",
+    "\n",
+    "print(tutor_q)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Randomized Question Types\n",
+    "\n",
+    "Prompt: Please generate a high-quality assessment consisting of {number of questions} varying questions, each of different types (open-ended, multiple choice, etc.), to determine if I achieved the following learning objectives in regards to {context}: {learning objectives}.\n",
+    "\n",
+    "Provide one question at a time, and wait for my response before providing me with feedback. Again, while the quiz may ask for multiple questions, you should only provide ONE question in you initial response. Do not include the answer in your response. If I get an answer wrong, provide me with an explanation of why it was incorrect,and then give me additional chances to respond until I get the correct choice. Explain why the correct choice is right."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Question 1 (Open-ended): Who is the narrator of the poem and what is the setting?\n",
+      "\n",
+      "Question 2 (Multiple choice): Which literary device is used in the line \"And sorry I could not travel both\"?\n",
+      "\n",
+      "a) Simile\n",
+      "b) Metaphor\n",
+      "c) Alliteration\n",
+      "d) Personification\n",
+      "\n",
+      "Question 3 (Short answer): Describe the underlying message of the poem in one sentence.\n",
+      "\n",
+      "Question 4 (Analogies): Complete the analogy: \"The two roads diverged in a yellow wood\" is to the physical setting as \"I took the one less traveled by\" is to ___________.\n",
+      "\n",
+      "Question 5 (Open-ended): Identify and explain one additional literary device used in the poem and its purpose.\n",
+      "\n",
+      "Please choose one question from above for me to provide a detailed evaluation.\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Randomized question types\n",
+    "tutor_q = get_tutoring_answer(context, mdl, assessment_request = ssp.SELF_STUDY_DEFAULTS['randomized_questions'],\n",
+    "                              learning_objectives = learning_objs)\n",
+    "\n",
+    "print(tutor_q)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Quantiative evaluation the correctness of a student's answer\n",
+    "\n",
+    "Prompt: (A continuation of the previous chat) Please generate the main points of the student’s answer to the previous question, and evaluate on a scale of 1 to 5 how comprehensive the student’s answer was in relation to the learning objectives, and explain why he or she received this rating, including what was missed in his or her answer if the student’s answer wasn’t complete.\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Based on the provided text, the student's answer to the previous question was not provided. Therefore, I cannot generate the main points of the student's answer or evaluate its comprehensiveness in relation to the learning objectives. Please provide the student's answer to the previous question so that I can assist you further.\n"
+     ]
+    }
+   ],
+   "source": [
+    "# qualitative evaluation\n",
+    "qualitative_query = \"\"\" Please generate the main points of the student’s answer to the previous question,\n",
+    " and evaluate on a scale of 1 to 5 how comprehensive the student’s answer was in relation to the learning objectives,\n",
+    " and explain why he or she received this rating, including what was missed in his or her answer if the student’s answer wasn’t complete.\"\"\"\n",
+    "\n",
+    "# Note that this uses the previous result and query in the context\n",
+    "last_answer = ''\n",
+    "\n",
+    "# Get result with formatting to emphasize changes in parameter inputs\n",
+    "result = get_tutoring_answer(last_answer + context,\n",
+    "                             mdl,\n",
+    "                             assessment_request = qualitative_query,\n",
+    "                             learning_objectives = learning_objs)\n",
+    "\n",
+    "print(result)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "python3",
+   "language": "python",
+   "name": "python3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}

lo-achievement/prompt_with_vector_store.ipynb

@@ -0,0 +1,637 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "<a href=\"https://colab.research.google.com/github/vanderbilt-data-science/lo-achievement/blob/main/prompt_with_vector_store.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# LLMs for Self-Study\n",
+    "> A prompt and code template for better understanding texts\n",
+    "\n",
+    "This notebook provides a guide for using LLMs for self-study programmatically. A number of prompt templates are provided to assist with generating great assessments for self-study, and code is additionally provided for fast usage. This notebook is best leveraged for a set of documents (text or PDF preferred) **to be uploaded** for interaction with the model.\n",
+    "\n",
+    "This version of the notebook is best suited for those who prefer to use files from their local drive as context rather than copy and pasting directly into the notebook to be used as context for the model. If you prefer to copy and paste text, you should direct yourself to the [prompt_with_context](https://colab.research.google.com/github/vanderbilt-data-science/lo-achievement/blob/main/prompt_with_context.ipynb) notebook."
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "skip_exec: true\n",
+    "---"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# run this code if you're using Google Colab or don't have these packages installed in your computing environment\n",
+    "! pip install pip install git+https://<token>@github.com/vanderbilt-data-science/lo-achievement.git"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#libraries for user setup code\n",
+    "from getpass import getpass\n",
+    "from logging import raiseExceptions\n",
+    "\n",
+    "#self import code\n",
+    "from ai_classroom_suite.PromptInteractionBase import *\n",
+    "from ai_classroom_suite.IOHelperUtilities import *\n",
+    "from ai_classroom_suite.SelfStudyPrompts import *\n",
+    "from ai_classroom_suite.MediaVectorStores import *"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# User Settings\n",
+    "In this section, you'll set your OpenAI API Key (for use with the OpenAI model), configure your environment/files for upload, and upload those files."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Run this cell and enter your OpenAI API key when prompted\n",
+    "set_openai_key()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Create model\n",
+    "mdl_name = 'gpt-3.5-turbo-16k'\n",
+    "chat_llm = create_model(mdl_name)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Define Your Documents Source\n",
+    "You may upload your files directly from your computer, or you may choose to do so via your Google Drive. Below, you will find instructions for both methods.\n",
+    "\n",
+    "For either model, begin by setting the `upload_setting` variable to:\n",
+    "* `'Local Drive'` - if you have files that are on your own computer (locally), or\n",
+    "* `'Google Drive'` - if you have files that are stored on Google Drive\n",
+    "\n",
+    "e.g.,\n",
+    "`upload_setting='Google Drive'`.\n",
+    "Don't forget the quotes around your selection!"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## Settings for upload: via local drive or Google Drive\n",
+    "### Please input either \"Google Drive\" or \"Local Drive\" into the empty string\n",
+    "\n",
+    "#upload_setting = 'Google Drive'\n",
+    "upload_setting = 'Local Drive'"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "<p style='color:green'><strong>Before Continuing</strong> - Make sure you have input your choice of upload into the `upload_setting`` variable above (Options: \"Local Drive\" or \"Google Drive\") as described in the above instructions.</p>"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Upload your Files\n",
+    "Now, you'll upload your files. When you run the below code cell, you'll be able to follow the instructions for local or Google Drive upload described here. If you would like to use our example document (Robert Frost's \"The Road Not Taken\", you can download the file from [this link](https://drive.google.com/drive/folders/1wpEoGACUqyNRYa4zBZeNkqcLJrGQbA53?usp=sharing) and upload via the instructions above.\n",
+    "\n",
+    "**If you selected **\"Local Drive\"** :**\n",
+    "> If you selected Local Drive, you'll need to start by selecting your local files. Run the code cell below. Once the icon appears, click the \"Choose File\". This will direct you to your computer's local drive. Select the file you would like to upload as context. The files will appear in the right sidebar. Then follow the rest of the steps in the \"Uploading Your files (Local Drive and Google Drive)\" below.\n",
+    "\n",
+    "**If you selected **\"Google Drive\"**: **\n",
+    "> If you selected Google Drive, you'll need to start by allowing access to your Google Drive. Run the code cell below. You will be redirected to a window where you will allow access to your Google Drive by logging into your Google Account. Your Drive will appear as a folder in the left side panel. Navigate through your Google Drive until you've found the file that you'd like to upload.\n",
+    "\n",
+    "Your files are now accessible to the code."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "e10a33b291a14f8089a1dea89f872998",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "FileChooser(path='/workspaces/lo-achievement', filename='', title='Use the following file chooser to add each …"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "4649cff2ba0942aa9c5e073be85f40fb",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Output()"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "# Run this cell then following the instructions to upload your file\n",
+    "selected_files = setup_drives(upload_setting)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['/workspaces/lo-achievement/roadnottaken.txt']"
+      ]
+     },
+     "execution_count": null,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "selected_files"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Resource and Personal Tutor Creation\n",
+    "Congratulations! You've nearly finished with the setup! From here, you can now run this section of cells using the arrow to the left to set up your vector store and create your model."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Create a vector store with your document\n",
+    "\n",
+    "With the file path, you can now create a vector store using the document that you uploaded. We expose this creation in case you want to modify the kind of vector store that you're creating. Run the cell below to create the default provided vector store."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Create vector store\n",
+    "doc_segments = get_document_segments(selected_files, data_type = 'files')\n",
+    "chroma_db, vs_retriever = create_local_vector_store(doc_segments, search_kwargs={\"k\": 1})"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Create the model which will do the vector store lookup and tutoring"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Create retrieval chain\n",
+    "qa_chain = create_tutor_mdl_chain(kind=\"retrieval_qa\", retriever = vs_retriever)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# A guide to prompting for self-study\n",
+    "In this section, we provide a number of different approaches for using AI to help you assess and explain the knowledge of your document. Start by interacting with the model and then try out the rest of the prompts!"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Brief overview of tutoring code options\n",
+    "\n",
+    "Now that your vector store is created, you can begin interacting with the model! You will interact with the model with a vector store using the `get_tutoring_answer` function below, and details are provided regarding the functionality below.\n",
+    "\n",
+    "Consider the multiple choice code snippet:\n",
+    "```{python}\n",
+    "tutor_q = get_tutoring_answer(context = '',\n",
+    "                              qa_chain,\n",
+    "                              assessment_request = SELF_STUDY_DEFAULTS['mc'],\n",
+    "                              learning_objectives = learning_objs,\n",
+    "                              input_kwargs = {'question':topic})\n",
+    "```\n",
+    "\n",
+    "This is how we're able to interact with the model for tutoring when using vector stores. The parameters are as follows:\n",
+    "\n",
+    "* `context` will be an empty string or you can also set it to `None`. This is because this field is automatically populated using the vector store retreiver.\n",
+    "* `qa_chain` is the model that you're using - we created this model chain a few cells above. \n",
+    "* `assessment_request` is your way of telling the model what kind of assessment you want. In the example above, we use some defaults provided for multiple choice. You can also insert your own text here. To learn more about these defaults, see the `prompt_with_context.ipynb` in the CLAS repo.\n",
+    "* `learning_objectives` are the learning objectives that you want to assess in a single paragraph string. You can set this to '' if you don't want to define any learning objectives. If you don't provide one, the model will use the default learning objectives.\n",
+    "* `input_kwargs` are additional inputs that we can define in the prompts. Above, you see that the keyword `question` is defined. `question` is the text used to retrieve relevant texts from the vector store. Above, we define a custom topic. If you were to omit this parameter, the model would use `assessment_request` as the text to retrieve relevant documents from the vector store. See the examples below for both scenarios.\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Sample topics and learning objectives\n",
+    "\n",
+    "Below, we define a topic (used to retrieve documents from the vector store if provided) and learning objectives which will be used in the following examples. You can change these as needed for your purpose."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Code topic\n",
+    "topic = 'The full text of the poem \"The Road Not Taken\" by Robert Frost'\n",
+    "\n",
+    "# set learning objectives if desired\n",
+    "learning_objs = (\"\"\"1. Identify the key elements of the work: important takeaways and underlying message.\n",
+    "                 2. Understand the literary devices used in prompting and in literature and their purpose.\"\"\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Types of Questions and Prompts\n",
+    "\n",
+    "Below is a comprehensive list of question types and prompt templates designed by our team. There are also example code blocks, where you can see how the model performed with the example and try it for yourself using the prompt template."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Multiple Choice\n",
+    "\n",
+    "Prompt: The following text should be used as the basis for the instructions which follow: {context}. Please design a 5 question quiz about {name or reference to context} which reflects the learning objectives: {list of learning objectives}. The questions should be multiple choice. If I get an answer wrong, provide me with an explanation of why it was incorrect, and then give me additional chances to respond until I get the correct choice. Explain why the correct choice is right."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Question 1: What is the underlying message of the excerpt?\n",
+      "\n",
+      "A) The speaker regrets not being able to travel both roads.\n",
+      "B) The speaker believes that taking the less traveled road has made a significant impact on their life.\n",
+      "C) The speaker is unsure about which road to choose.\n",
+      "D) The speaker is fascinated by the beauty of the yellow wood.\n",
+      "\n",
+      "Please select one of the options (A, B, C, or D) and provide your answer.\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Multiple choice code example\n",
+    "tutor_q = get_tutoring_answer('', qa_chain, assessment_request = SELF_STUDY_DEFAULTS['mc'],\n",
+    "                              learning_objectives = learning_objs, input_kwargs = {'question':topic})\n",
+    "\n",
+    "print(tutor_q)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Short Answer\n",
+    "\n",
+    "Prompt: Please design a 5-question quiz about {context} which reflects the learning objectives: {list of learning objectives}. The questions should be short answer. Expect the correct answers to be {anticipated length} long. If I get any part of the answer wrong, provide me with an explanation of why it was incorrect, and then give me additional chances to respond until I get the correct choice."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Question 1: What is the underlying message of the poem?\n",
+      "\n",
+      "Remember to provide your answer in a few sentences.\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Short answer code example\n",
+    "tutor_q = get_tutoring_answer(None, qa_chain, assessment_request = SELF_STUDY_DEFAULTS['short_answer'],\n",
+    "                              learning_objectives = learning_objs, input_kwargs = {'question':topic})\n",
+    "\n",
+    "print(tutor_q)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Fill-in-the-blank\n",
+    "\n",
+    "Prompt: Create a 5 question fill in the blank quiz refrencing {context}. The quiz should reflect the learning objectives: {learning objectives}. Please prompt me one question at a time and proceed when I answer correctly. If I answer incorrectly, please explain why my answer is incorrect.\n",
+    "\n",
+    ":::{.callout-info}\n",
+    "In the example below, we omit the `input_kwargs` parameter. This means we'll use the text from `assessment_request` as the question topic.\n",
+    ":::"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Question: The speaker in the poem \"The Road Not Taken\" is faced with a choice between _______ roads.\n",
+      "\n",
+      "Please provide your answer.\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Fill in the blank code example\n",
+    "tutor_q = get_tutoring_answer(None, qa_chain, assessment_request = SELF_STUDY_DEFAULTS['fill_blank'],\n",
+    "                              learning_objectives = learning_objs)\n",
+    "\n",
+    "print(tutor_q)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Sequencing\n",
+    "\n",
+    "Prompt: Please develop a 5 question questionnaire that will ask me to recall the steps involved in the following learning objectives in regard to {context}: {learning objectives}. After I respond, explain their sequence to me."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Question 1: What is the underlying message or theme of the provided text?\n",
+      "\n",
+      "(Note: Please provide your response and I will evaluate it.)\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Sequence example\n",
+    "tutor_q = get_tutoring_answer(None, qa_chain, assessment_request = SELF_STUDY_DEFAULTS['sequencing'],\n",
+    "                              learning_objectives = learning_objs)\n",
+    "\n",
+    "print(tutor_q)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Relationships/drawing connections\n",
+    "\n",
+    "Prompt: Please design a 5 question quiz that asks me to explain the relationships that exist within the following learning objectives, referencing {context}: {learning objectives}."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Question 1: What is the underlying message or theme of the text \"The Road Not Taken\"?\n",
+      "\n",
+      "(Note: The answer to this question will require the student to identify the key elements and important takeaways from the text in order to determine the underlying message or theme.)\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Relationships example\n",
+    "tutor_q = get_tutoring_answer(None, qa_chain, assessment_request = SELF_STUDY_DEFAULTS['relationships'],\n",
+    "                              learning_objectives = learning_objs)\n",
+    "\n",
+    "print(tutor_q)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Concepts and Definitions\n",
+    "\n",
+    "Prompt: Design a 5 question quiz that asks me about definitions related to the following learning objectives: {learning objectives} - based on {context}\".\n",
+    "Once I write out my response, provide me with your own response, highlighting why my answer is correct or incorrect."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Question 1: Based on the provided text, what is the underlying message or theme of the work?\n",
+      "\n",
+      "Please provide your response.\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Concepts and definitions example\n",
+    "tutor_q = get_tutoring_answer(None, qa_chain, assessment_request = SELF_STUDY_DEFAULTS['concepts'],\n",
+    "                              learning_objectives = learning_objs)\n",
+    "\n",
+    "print(tutor_q)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Real Word Examples\n",
+    "\n",
+    "Prompt: Demonstrate how {context} can be applied to solve a real-world problem related to the following learning objectives: {learning objectives}. Ask me questions regarding this theory/concept."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Based on the provided context, it seems that the extracted text is a poem by Robert Frost and does not directly provide any information or context related to problem-solving in the real world. Therefore, it may not be possible to demonstrate how the provided context can be applied to solve a real-world problem. However, I can still assess your understanding of the learning objectives mentioned. Let's start with the first learning objective: identifying the key elements of the work, important takeaways, and underlying message. \n",
+      "\n",
+      "Question 1: Based on your reading of the poem, what are some key elements or important takeaways that you can identify?\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Real word example\n",
+    "tutor_q = get_tutoring_answer(None, qa_chain, assessment_request = SELF_STUDY_DEFAULTS['real_world_example'],\n",
+    "                              learning_objectives = learning_objs)\n",
+    "\n",
+    "print(tutor_q)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Randomized Question Types\n",
+    "\n",
+    "Prompt: Please generate a high-quality assessment consisting of 5 varying questions, each of different types (open-ended, multiple choice, etc.), to determine if I achieved the following learning objectives in regards to {context}: {learning objectives}. If I answer incorrectly for any of the questions, please explain why my answer is incorrect."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Question 1 (Open-ended):\n",
+      "Based on the given excerpt, what do you think is the underlying message or theme of the text? Please provide a brief explanation to support your answer.\n",
+      "\n",
+      "(Note: The answer to this question will vary depending on the student's interpretation of the text. As the tutor, you can provide feedback on the strengths and weaknesses of their response, and guide them towards a deeper understanding of the text's message.)\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Randomized question types\n",
+    "tutor_q = get_tutoring_answer(None, qa_chain, assessment_request = SELF_STUDY_DEFAULTS['randomized_questions'],\n",
+    "                              learning_objectives = learning_objs)\n",
+    "\n",
+    "print(tutor_q)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Quantiative evaluation the correctness of a student's answer\n",
+    "\n",
+    "Prompt: (A continuation of the previous chat) Please generate the main points of the student’s answer to the previous question, and evaluate on a scale of 1 to 5 how comprehensive the student’s answer was in relation to the learning objectives, and explain why he or she received this rating, including what was missed in his or her answer if the student’s answer wasn’t complete.\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Main points of the student's answer:\n",
+      "- The underlying message of the text is that people should follow the crowd and take the easy way instead of the road less traveled.\n",
+      "- The road less traveled is hard and painful to traverse.\n",
+      "\n",
+      "Evaluation of the student's answer:\n",
+      "I would rate the student's answer a 2 out of 5 in terms of comprehensiveness in relation to the learning objectives. \n",
+      "\n",
+      "Explanation:\n",
+      "The student correctly identifies that the underlying message of the text is related to choosing between two paths, but their interpretation of the message is not entirely accurate. The student suggests that the text encourages people to follow the crowd and take the easy way, which is not supported by the actual message of the poem. The poem actually suggests that taking the road less traveled can make a significant difference in one's life. The student also mentions that the road less traveled is hard and painful to traverse, which is not explicitly stated in the text. This interpretation may be influenced by the student's personal perspective rather than the actual content of the poem. Therefore, the student's answer is not complete and does not fully grasp the intended message of the text.\n"
+     ]
+    }
+   ],
+   "source": [
+    "# qualitative evaluation\n",
+    "qualitative_query = \"\"\" Please generate the main points of the student’s answer to the previous question,\n",
+    " and evaluate on a scale of 1 to 5 how comprehensive the student’s answer was in relation to the learning objectives,\n",
+    " and explain why he or she received this rating, including what was missed in his or her answer if the student’s answer wasn’t complete.\"\"\"\n",
+    "\n",
+    "last_answer = (\"TUTOR QUESTION: Question 1 (Open-ended): \" +\n",
+    "               \"Based on the given excerpt, what do you think is the underlying message or theme of the text? Please provide a \" + \n",
+    "               \"brief explanation to support your answer.\\n\" + \n",
+    "               \"STUDENT ANSWER: The underlying message of the text is that people should follow the crowd and the road less traveled is hard \"+\n",
+    "               \"and painful to traverse. Take the easy way instead. \")\n",
+    "\n",
+    "# Note that this uses the previous result and query in the context\n",
+    "tutor_q = get_tutoring_answer(None, qa_chain, assessment_request = qualitative_query + '\\n' + last_answer,\n",
+    "                              learning_objectives = learning_objs,\n",
+    "                              input_kwargs = {'question':topic})\n",
+    "\n",
+    "print(tutor_q)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "python3",
+   "language": "python",
+   "name": "python3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}

lo-achievement/settings.ini

@@ -0,0 +1,43 @@
+[DEFAULT]
+# All sections below are required unless otherwise specified.
+# See https://github.com/fastai/nbdev/blob/master/settings.ini for examples.
+
+### Python library ###
+repo = lo-achievement
+lib_name = ai_classroom_suite
+version = 0.0.1
+min_python = 3.7
+license = apache2
+black_formatting = False
+
+### nbdev ###
+doc_path = _docs
+lib_path = ai_classroom_suite
+nbs_path = nbs
+recursive = True
+tst_flags = notest
+put_version_in_init = True
+
+### Docs ###
+branch = main
+custom_sidebar = False
+doc_host = https://%(user)s.github.io
+doc_baseurl = /%(repo)s
+git_url = https://github.com/%(user)s/%(repo)s
+title = %(lib_name)s
+
+### PyPI ###
+audience = Developers
+author = Charreau Bell
+author_email = [email protected]
+copyright = 2023 onwards, %(author)s
+description = A repository supporting enhanced instruction and grading using AI
+keywords = nbdev jupyter notebook python
+language = English
+status = 3
+user = vanderbilt-data-science
+
+### Optional ###
+requirements = langchain pandas numpy getpass openai gradio chromadb tiktoken unstructured pdf2image yt_dlp libmagic chromadb librosa deeplake ipyfilechooser
+# dev_requirements = 
+# console_scripts =

lo-achievement/setup.py

@@ -0,0 +1,57 @@
+from pkg_resources import parse_version
+from configparser import ConfigParser
+import setuptools, shlex
+assert parse_version(setuptools.__version__)>=parse_version('36.2')
+
+# note: all settings are in settings.ini; edit there, not here
+config = ConfigParser(delimiters=['='])
+config.read('settings.ini', encoding='utf-8')
+cfg = config['DEFAULT']
+
+cfg_keys = 'version description keywords author author_email'.split()
+expected = cfg_keys + "lib_name user branch license status min_python audience language".split()
+for o in expected: assert o in cfg, "missing expected setting: {}".format(o)
+setup_cfg = {o:cfg[o] for o in cfg_keys}
+
+licenses = {
+    'apache2': ('Apache Software License 2.0','OSI Approved :: Apache Software License'),
+    'mit': ('MIT License', 'OSI Approved :: MIT License'),
+    'gpl2': ('GNU General Public License v2', 'OSI Approved :: GNU General Public License v2 (GPLv2)'),
+    'gpl3': ('GNU General Public License v3', 'OSI Approved :: GNU General Public License v3 (GPLv3)'),
+    'bsd3': ('BSD License', 'OSI Approved :: BSD License'),
+}
+statuses = [ '1 - Planning', '2 - Pre-Alpha', '3 - Alpha',
+    '4 - Beta', '5 - Production/Stable', '6 - Mature', '7 - Inactive' ]
+py_versions = '3.6 3.7 3.8 3.9 3.10'.split()
+
+requirements = shlex.split(cfg.get('requirements', ''))
+if cfg.get('pip_requirements'): requirements += shlex.split(cfg.get('pip_requirements', ''))
+min_python = cfg['min_python']
+lic = licenses.get(cfg['license'].lower(), (cfg['license'], None))
+dev_requirements = (cfg.get('dev_requirements') or '').split()
+
+setuptools.setup(
+    name = cfg['lib_name'],
+    license = lic[0],
+    classifiers = [
+        'Development Status :: ' + statuses[int(cfg['status'])],
+        'Intended Audience :: ' + cfg['audience'].title(),
+        'Natural Language :: ' + cfg['language'].title(),
+    ] + ['Programming Language :: Python :: '+o for o in py_versions[py_versions.index(min_python):]] + (['License :: ' + lic[1] ] if lic[1] else []),
+    url = cfg['git_url'],
+    packages = setuptools.find_packages(),
+    include_package_data = True,
+    install_requires = requirements,
+    extras_require={ 'dev': dev_requirements },
+    dependency_links = cfg.get('dep_links','').split(),
+    python_requires  = '>=' + cfg['min_python'],
+    long_description = open('README.md', encoding='utf-8').read(),
+    long_description_content_type = 'text/markdown',
+    zip_safe = False,
+    entry_points = {
+        'console_scripts': cfg.get('console_scripts','').split(),
+        'nbdev': [f'{cfg.get("lib_path")}={cfg.get("lib_path")}._modidx:d']
+    },
+    **setup_cfg)
+
+