Add automatic release generation system so dependencies will be updated per release
This commit is contained in:
James Taylor
parent
3d57e14df7
commit
7b42d26c61
6
.gitignore
vendored
6
.gitignore
vendored
@ -3,7 +3,11 @@ __pycache__/
|
||||
debug/
|
||||
data/
|
||||
python/
|
||||
release/
|
||||
youtube-local/
|
||||
banned_addresses.txt
|
||||
settings.txt
|
||||
|
||||
get-pip.py
|
||||
latest-dist.zip
|
||||
*.zip
|
||||
*venv*
|
||||
|
||||
196
generate_release.py
Normal file
196
generate_release.py
Normal file
@ -0,0 +1,196 @@
|
||||
# Generate a windows release and a generated embedded distribution of python
|
||||
# Latest python version is the argument of the script
|
||||
# Only works on windows at the moment
|
||||
|
||||
import sys
|
||||
import urllib
|
||||
import urllib.request
|
||||
import subprocess
|
||||
import shutil
|
||||
import os
|
||||
|
||||
latest_version = sys.argv[1]
|
||||
if sys.argv[2] == '-nd':
|
||||
downloads_enabled = False
|
||||
elif sys.argv[2] == '-d':
|
||||
downloads_enabled = True
|
||||
else:
|
||||
raise Exception('No download switch specified')
|
||||
|
||||
def check(code):
|
||||
if code != 0:
|
||||
raise Exception('Got nonzero exit code from command')
|
||||
|
||||
def log(line):
|
||||
print('[generate_release.py] ' + line)
|
||||
|
||||
# https://stackoverflow.com/questions/7833715/python-deleting-certain-file-extensions
|
||||
def remove_files_with_extensions(path, extensions):
|
||||
for root, dirs, files in os.walk(path):
|
||||
for file in files:
|
||||
if os.path.splitext(file)[1] in extensions:
|
||||
os.remove(os.path.join(root, file))
|
||||
|
||||
# ---------- Get current release version, for later ----------
|
||||
log('Getting current release version')
|
||||
describe_result = subprocess.run(['git', 'describe', '--tags'], stdout=subprocess.PIPE)
|
||||
if describe_result.returncode != 0:
|
||||
raise Exception('Git describe failed')
|
||||
|
||||
release_tag = describe_result.stdout.strip().decode('ascii')
|
||||
|
||||
|
||||
# ----------- Make copy of youtube-local files using git -----------
|
||||
|
||||
if os.path.exists('./youtube-local'):
|
||||
log('Removing old release')
|
||||
shutil.rmtree('./youtube-local')
|
||||
|
||||
# Export git repository - this will ensure .git and things in gitignore won't
|
||||
# be included. Git only supports exporting archive formats, not into
|
||||
# directories, so pipe into 7z to put it into .\youtube-local (not to be
|
||||
# confused with working directory. I'm calling it the same thing so it will
|
||||
# have that name when extracted from the final release zip archive)
|
||||
log('Making copy of youtube-local files')
|
||||
check(os.system('git archive --format tar master | 7z x -si -ttar -oyoutube-local'))
|
||||
|
||||
if len(os.listdir('./youtube-local')) == 0:
|
||||
raise Exception('Failed to copy youtube-local files')
|
||||
|
||||
|
||||
# ----------- Generate embedded python distribution -----------
|
||||
os.environ['PYTHONDONTWRITEBYTECODE'] = '1' # *.pyc files double the size of the distribution
|
||||
get_pip_url = 'https://bootstrap.pypa.io/get-pip.py'
|
||||
latest_dist_url = 'https://www.python.org/ftp/python/' + latest_version + '/python-' + latest_version + '-embed-win32.zip'
|
||||
|
||||
if downloads_enabled:
|
||||
log('Downloading get-pip.py...')
|
||||
get_pip = urllib.request.urlopen(get_pip_url).read()
|
||||
log('Finished downloading get-pip.py')
|
||||
|
||||
with open('./get-pip.py', 'wb') as f:
|
||||
f.write(get_pip)
|
||||
|
||||
log('Downloading latest python distribution...')
|
||||
latest_dist= urllib.request.urlopen(latest_dist_url).read()
|
||||
log('Finished downloading python distribution')
|
||||
|
||||
with open('./latest-dist.zip', 'wb') as f:
|
||||
f.write(latest_dist)
|
||||
|
||||
if os.path.exists('./python'):
|
||||
log('Removing old python distribution')
|
||||
shutil.rmtree('./python')
|
||||
|
||||
|
||||
log('Extracting python distribution')
|
||||
|
||||
check(os.system(r'7z -y x -opython latest-dist.zip'))
|
||||
|
||||
log('Executing get-pip.py')
|
||||
os.system(r'.\python\python.exe -I get-pip.py')
|
||||
|
||||
'''
|
||||
# Explanation of .pth, ._pth, and isolated mode
|
||||
|
||||
## Isolated mode
|
||||
We want to run in what is called isolated mode, given by the switch -I.
|
||||
This mode prevents the embedded python distribution from searching in
|
||||
global directories for imports
|
||||
|
||||
For example, if a user has `C:\Python37` and the embedded distribution is
|
||||
the same version, importing something using the embedded distribution will
|
||||
search `C:\Python37\Libs\site-packages`. This is not desirable because it
|
||||
means I might forget to distribute a dependency if I have it installed
|
||||
globally and I don't see any import errors. It also means that an outdated
|
||||
package might override the one being distributed and cause other problems.
|
||||
|
||||
Isolated mode also means global environment variables and registry
|
||||
entries will be ignored
|
||||
|
||||
## The trouble with isolated mode
|
||||
Isolated mode also prevents the current working directory (cwd) from
|
||||
being added to `sys.path`. `sys.path` is the list of directories python will
|
||||
search in for imports. In non-isolated mode this is automatically populated
|
||||
with the cwd, `site-packages`, the directory of the python executable, etc.
|
||||
|
||||
# How to get the cwd into sys.path in isolated mode
|
||||
The hack to get this to work is to use a .pth file. Normally, these files
|
||||
are just an additional list of directories to be added to `sys.path`.
|
||||
However, they also allow arbitrary code execution on lines beginning with
|
||||
`import ` (see https://docs.python.org/3/library/site.html). So, we simply
|
||||
add `import sys; sys.path.insert(0, '')` to add the cwd to path. `''` is
|
||||
shorthand for the cwd. See https://bugs.python.org/issue33698#msg318272
|
||||
|
||||
# ._pth files in the embedded distribution
|
||||
A python37._pth file is included in the embedded distribution. The presence
|
||||
of tis file causes the embedded distribution to always use isolated mode
|
||||
(which we want). They are like .pth files, except they do not allow the
|
||||
arbitrary code execution trick. In my experimentation, I found that they
|
||||
prevent .pth files from loading. So the ._pth file will have to be removed
|
||||
and replaced with a .pth. Isolated mode will have to be specified manually.
|
||||
'''
|
||||
|
||||
log('Removing ._pth')
|
||||
major_release = latest_version.split('.')[1]
|
||||
os.remove(r'.\python\python3' + major_release + '._pth')
|
||||
|
||||
log('Adding path_fixes.pth')
|
||||
with open(r'.\python\path_fixes.pth', 'w', encoding='utf-8') as f:
|
||||
f.write("import sys; sys.path.insert(0, '')\n")
|
||||
|
||||
|
||||
'''# python3x._pth file tells the python executable where to look for files
|
||||
# Need to add the directory where packages are installed,
|
||||
# and the parent directory (which is where the youtube-local files are)
|
||||
major_release = latest_version.split('.')[1]
|
||||
with open('./python/python3' + major_release + '._pth', 'a', encoding='utf-8') as f:
|
||||
f.write('.\\Lib\\site-packages\n')
|
||||
f.write('..\n')'''
|
||||
|
||||
log('Inserting Microsoft C Runtime')
|
||||
check(os.system(r'copy C:\Windows\SysWOW64\msvcp140.dll .\python\msvcp140.dll'))
|
||||
|
||||
log('Installing dependencies')
|
||||
check(os.system(r'.\python\python.exe -I -m pip install --no-compile -r .\requirements.txt'))
|
||||
|
||||
log('Uninstalling unnecessary gevent stuff')
|
||||
check(os.system(r'.\python\python.exe -I -m pip uninstall --yes cffi pycparser'))
|
||||
shutil.rmtree(r'./python/Lib/site-packages/gevent/tests')
|
||||
shutil.rmtree(r'./python/Lib/site-packages/gevent/testing')
|
||||
remove_files_with_extensions(r'./python/Lib/site-packages/gevent', ['.html']) # bloated html documentation
|
||||
|
||||
log('Uninstalling pip and others')
|
||||
check(os.system(r'.\python\python.exe -I -m pip uninstall --yes pip setuptools wheel'))
|
||||
|
||||
log('Removing pyc files') # Have to do this because get-pip and some packages don't respect --no-compile
|
||||
remove_files_with_extensions(r'./python', ['.pyc'])
|
||||
|
||||
log('Removing dist-info and __pycache__')
|
||||
for root, dirs, files in os.walk(r'./python'):
|
||||
for dir in dirs:
|
||||
if dir == '__pycache__' or dir.endswith('.dist-info'):
|
||||
shutil.rmtree(os.path.join(root, dir))
|
||||
|
||||
|
||||
'''log('Removing get-pip.py and zipped distribution')
|
||||
os.remove(r'.\get-pip.py')
|
||||
os.remove(r'.\latest-dist.zip')'''
|
||||
|
||||
print()
|
||||
log('Finished generating python distribution')
|
||||
|
||||
# ----------- Copy generated distribution into release folder -----------
|
||||
log('Copying python distribution into release folder')
|
||||
shutil.copytree(r'.\python', r'.\youtube-local\python')
|
||||
|
||||
# ----------- Create release zip -----------
|
||||
output_filename = 'youtube-local-' + release_tag + '-windows.zip'
|
||||
if os.path.exists('./' + output_filename):
|
||||
log('Removing previous zipped release')
|
||||
os.remove('.\\' + output_filename)
|
||||
log('Zipping release')
|
||||
check(os.system(r'7z -mx=9 a ' + output_filename + ' .\youtube-local'))
|
||||
|
||||
print('\n')
|
||||
log('Finished')
|
||||
Loading…
x
Reference in New Issue
Block a user