# default_exp html
⚠️ This writing is a work in progress.⚠️
Please read everything found on the mainpage before continuing; disclaimer and all.


About this Tutorial:
note: There is a lot of quoting and paraphrasing on this page in particular. Be sure to check the links you want to learn more.
We can update it in-drive and then rerun the results in near-real-time.
Several post-render functions will be used to reduce network requests.
I intend to hook this up to our webpack scripts for all the benefits.
Config Env
!pip install marko
!python -m pip install nbdev#export
import re
from IPython.display import clear_output
import marko#export
from nbdev import export2html
from nbdev.export2html import read_nb, get_metadata, clean_exports
from nbdev.export import split_flags_and_code, check_re_multifrom IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = "all"
FNS
These scripts will convert an nb to an html doc. NBdev and Marko among other libraries are used.
#export
# 1st fn. passes each cell to decision fn.
def convertNb(nb): return [cleanCell(c) for c in nb['cells']]
# 2nd fn. returns text or passes cell to 'code cell' processor
def cleanCell(cell):
if cell.cell_type == 'markdown':
source = cell.source
if ( re.search('
', source) ): return source.replace('\r\n', "source")
return marko.convert(source)
else: return processCode(cell)
# 3rd fn. Calls getFlags, processSource, processOutput
def processCode(cell):
x = [cell.source]
flags = getFlags(x[0])
x[0] = processSource( x[0], flags )
if ( len(cell.outputs) ):
for o in cell.outputs: x.append( processOutput(o, flags) )
return x#export
def runit(fname='index.ipynb'):
# fname = 'index.ipynb'
nb = read_nb(fname)
meta_jekyll = get_metadata(nb['cells'])
outp = convertNb(nb)
txt = ''.join( [''.join(c) for c in outp] )
return txtMisc Notes
Unsorted notes from previous nb
"nb.keys()"'nb.keys()'
- display_data = ['text/html', 'text/plain']
- stream
- execute_result = ['image/jpeg', 'text/plain']
- error
%html
- {'output_type': 'display_data',
'data': {'text/html': <script console.log('hi')\n script>",
'text/plain': '<IPython.core.display.HTML object>'} }
raw markdown
- {'output_type': 'stream', 'text': 'Thisis text \n'}
display( airbnb.head() ); Image(filename='test.jpg')
- {'output_type': 'display_data', 'data': {'text/html': div>oooAKKKKAP/Z</div', 'text/plain': '<IPython.core.display.Image object>'}
Convert FN's
This next bit will run the conversion on a single file.
import IPython
import requests
from google.colab import output
t= """
outp = runit()
v = 0 if not 'v' in globals() else v+1
# page = f"index{v}.html"
page = f"index.html"
print(page)
f = open(page, "a")
f.write( ''.join( [''.join(c) for c in outp] ) )
f.close()
# posted = requests.get( 'https://charleskarpati.com/version.php', params=[('v', v)], ).json()
# IPython.display.HTML( outp )
"""Serve Files with Ngrok
You can test on the Colab by using a flask server which can be a bit tricky.
from flask_ngrok import run_with_ngrok
from flask import Flask
from numpy import random
import requests
app = Flask(__name__)
run_with_ngrok(app) #starts ngrok when the app is run
def getHtml(filepath):
with open(filepath, 'r') as file:
data = file.read().replace('\n', '')
return data
@app.route("/")
def index():
# Get version number using the localstorage.
# v = requests.get( 'https://charleskarpati.com/version.php', ).json()
# location = f'index{v}.html'
# print('\n \n LOCATION: ', location)
# return getHtml(location)
outp = runit('BniaScripts.ipynb')
resp = ''.join( [''.join(c) for c in outp] )
resp = resp.replace('', '')
return resp
@app.route("/about")
def about():
return getHtml('lines.html')
app.run()Working with GDrive Directories, Github, NBDev
Connect to Drive
If you have content on your Google Drive, you can publish entire directorys.
#hide
!pip install nbdev
!pip install flask_ngrok
from google.colab import drive
drive.mount('/content/drive')
%cd /content/drive/My Drive/'Software Development Documents'/dataplayDrive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount("/content/drive", force_remount=True).
/content/drive/My Drive/Software Development Documents/dataplay
project = 'datalabs' # 'dataguide' 'datalabs' 'dataplay' 'VitalSigns', 'DevelopersDocumentation'
%cd ../{project}/ /content/drive/My Drive/Software Development Documents/datalabs
import requests
from IPython.core.display import display, HTMLSETUP
addToHeader = """ """
def addAfterNav(title): return f""" """
addToFooter = """
"""nav = navTop + navMid + navBottom
Run one
ls[0m[01;34mbuild[0m/ [01;34mdist[0m/ LICENSE MANIFEST.in README.md setup.py
CONTRIBUTING.md [01;34mdocs[0m/ Makefile [01;34mnotebooks[0m/ settings.ini
%cd notebooks/content/drive/My Drive/Software Development Documents/dataguide/notebooks
ls01_Github.ipynb 07_Javascript_Packages.ipynb
02_Colabs.ipynb 08_Cloud_Functions.ipynb
03_Shell_Basics.ipynb 09_Browser_Extensions.ipynb
04_APIs.ipynb [0m[01;34mimages[0m/
05_Webscraping.ipynb index.ipynb
06_Nbdev.ipynb
from flask_ngrok import run_with_ngrok
from flask import Flask
from numpy import random
import requests
header = requests.get('https://bniajfi.org/wp-json/wp/v2/header')
header = header.content.decode('utf-8')[:-4]
header= header.replace('/wp-content/uploads/2021/02/bnia_text.png', 'https://bniajfi.org/wp-content/uploads/2021/02/bnia_text.png')
header = header + addToHeader + nav
footer = requests.get('https://bniajfi.org/wp-json/wp/v2/footer')
footer = addToFooter+footer.content.decode('utf-8')[:-4]
app = Flask(__name__)
run_with_ngrok(app) #starts ngrok when the app is run
o = """
def getHtml(filepath):
with open(filepath, 'r') as file:
data = file.read().replace('\n', '')
return data
"""
@app.route("/")
def index():
def template(title, x): return f'{header}{title}{x}{footer}'
def replace(txt):
txt = txt.replace('', '')
return txt
# Get version number using the localstorage.
# v = requests.get( 'https://charleskarpati.com/version.php', ).json()
# location = f'index{v}.html'
# print('\n \n LOCATION: ', location)
# return getHtml(location)
infilename = 'index.ipynb'
outp = runit(infilename)
title = "".join(re.findall('[a-zA-Z][^a-zA-Z]*', 'index.ipynb'[:-6]))
resp = ''.join( [''.join(c) for c in outp] )
resp = replace(resp)
resp = template( addAfterNav(title) , resp)
return resp
@app.route("/about")
def about():
return getHtml('lines.html')
app.run()
Run all
ls[0m[01;34mbuild[0m/ [01;34mdist[0m/ Makefile settings.ini
CONTRIBUTING.md [01;34mdocs[0m/ MANIFEST.in setup.py
[01;34mdatalabs.egg-info[0m/ LICENSE [01;34mnotebooks[0m/
!git rm -r docs -frm 'docs/Courts_MechanicalSoup.html'
rm 'docs/amivi_meetup.html'
rm 'docs/index.html'
rm 'docs/nb_2_html_tests.html'
rm 'docs/nbdev_create_clean_publish.html'
rm 'docs/scooter_exploration.html'
rm 'docs/web_scraper.html'
rm 'docs/wordclouds.html'
!rm docs -rrm: cannot remove 'docs': No such file or directory
mkdir docsls[0m[01;34mbuild[0m/ [01;34mdist[0m/ Makefile settings.ini
CONTRIBUTING.md [01;34mdocs[0m/ MANIFEST.in setup.py
[01;34mdatalabs.egg-info[0m/ LICENSE [01;34mnotebooks[0m/
# ! nbdev_build_docs --fname '' # Makes the readme.md
import os
def template(title, x): return f'{header}{title}{x}{footer}'
def replace(txt):
txt = txt.replace('', '')
return txt
# traverse whole directory
for file in os.listdir(r'notebooks'):
# check the extension of files
if (file.endswith('.ipynb') ):
# print whole path of files
print(os.path.join('notebooks', file))
title = "".join(re.findall('[a-zA-Z][^a-zA-Z]*', file[:-6]))
page = f"./docs/{title}.html"
outp = runit(os.path.join('notebooks', file))
resp = ''.join( [''.join(c) for c in outp] )
resp = replace(resp)
resp = template( addAfterNav(title) , resp)
try: f = open(page, "a")
except: f = open(page[1:], "a")
f.write( resp )
f.close()
# send it to my websesrver for saving.! nbdev_build_docs --fname '' --mk_readme TRUENo notebooks were modified
converting /content/drive/My Drive/Software Development Documents/dataguide/notebooks/index.ipynb to README.md
Publishing a new Repo
lsCONTRIBUTING.md [0m[01;34mindex_files[0m/ [01;34mnotebooks[0m/ [01;34mVitalSigns[0m/
[01;34mdist[0m/ LICENSE README.md [01;34mVitalSigns.egg-info[0m/
docker-compose.yml Makefile settings.ini
[01;34mdocs[0m/ MANIFEST.in setup.py
#step 1
! git init
! git config --global user.email "bniajfi@gmail.com"
! git config --global user.name "bniajfi"Initialized empty Git repository in /content/drive/My Drive/Software Development Documents/VitalSigns/.git/
#step 2
! git add *
! git commit -m "Second commit for styling"
! git branch -M main # https://github.com/settings/tokens# step 3
token = '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'
username = 'BNIA'
print(f'https://{token}@github.com/{username}/{project}.git')
! git remote remove origin
! git remote add origin 'https://{token}@github.com/{username}/{project}.git'
#! git push -u origin main
!git push 'https://{token}@github.com/{username}/{project}.git' https://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@github.com/BNIA/VitalSigns.git
fatal: No such remote: origin
Counting objects: 92, done.
Delta compression using up to 2 threads.
Compressing objects: 100% (91/91), done.
Writing objects: 100% (92/92), 640.40 KiB | 2.10 MiB/s, done.
Total 92 (delta 35), reused 0 (delta 0)
remote: Resolving deltas: 100% (35/35), done.[K
To https://github.com/BNIA/VitalSigns.git
* [new branch] main -> main
Pypi
lsCONTRIBUTING.md [0m[01;34mindex_files[0m/ [01;34mnotebooks[0m/ [01;34mVitalSigns[0m/
[01;34mdist[0m/ LICENSE README.md [01;34mVitalSigns.egg-info[0m/
docker-compose.yml Makefile settings.ini
[01;34mdocs[0m/ MANIFEST.in setup.py
! pip install twine
! nbdev_bump_version
! make pypils[0m[01;34mbuild[0m/ [01;34mdataplay.egg-info[0m/ LICENSE [01;34mnotebooks[0m/ setup.py
CONTRIBUTING.md [01;34mdist[0m/ Makefile README.md
[01;34mdataplay[0m/ [01;34mdocs[0m/ MANIFEST.in settings.ini
#hide
# https://nbdev.fast.ai/tutorial.html#Add-in-notebook-export-cell
# https://nbdev.fast.ai/sync#nbdev_update_lib
# first. builds the .py files from from .ipynbs
# !nbdev_build_lib # --fname filename.ipynb
# second. Push .py changes back to their original .ipynbs
# !nbdev_update_lib
# sometimes. Update .ipynb import statements if the .py filename.classname changes.
# !relimport2name
# nbdev_build_docs builds the documentation from the notebooks
# !nbdev_build_docs --force_all True --mk_readme True
The “src refspec master does not match any” error occurs if you have forgotten to add the files you have changed to a commit and try to push those changes to a remote repository before you make the first commit in your repository.
'https://docs.github.com/en/github/importing-your-projects-to-github/importing-source-code-to-github/adding-an-existing-project-to-github-using-the-command-line'
'https://stackoverflow.com/questions/61424599/error-when-pushing-files-to-git-in-colab-fatal-could-not-read-username-for-ht'
'https://stackoverflow.com/questions/492558/removing-multiple-files-from-a-git-repo-that-have-already-been-deleted-from-disk'
'https://github.community/t/error-refname-refs-heads-master-not-found/144737/13'
'https://github.com/settings/tokens'
'https://github.com/settings/tokens'
'https://stackoverflow.com/questions/61424599/error-when-pushing-files-to-git-in-colab-fatal-could-not-read-username-for-ht'

headerPHPEndsLike = """