Commit 979865d5 authored by Pratap Vardhan's avatar Pratap Vardhan

ENH: builderrors

parents
Pipeline #92129 failed with stage
in 24 seconds
TODO
# Ignore auto-generated documentation
docs/gramex*.rst
docs/modules.rst
# Ignore files generated by testcases
tests/**/*.test
tests/**/gen.*
testlib/**/gen.*
# Ignore byte-compiled / optimised / DLL files
*.py[cod]
# Filenames should NOT have spaces
* *
# Ignore SQLite3 files. Gramex creates some automatically
*.sqlite3
*.sqlite3-journal
# Cache folders used for testing
.cache-*
.pytest_cache
# Ignore log files
*.log*
# Don't commit data files, except what's required for testing or by Gramex apps
*.csv
*.xls*
!tests/**/*.csv
!tests/*.xlsx
!testlib/**/*.csv
!testlib/*.xlsx
!gramex/apps/**/*.csv
!gramex/apps/guide/**/*.xlsx
*.ppt*
!testlib/input.pptx
!tests/template.pptx
!gramex/apps/guide/formhandler/input.pptx
!gramex/apps/guide/pptxhandler/examples-input.pptx
# Don't commit databases created by test cases
tests/*.db
# Don't commit uploads created by test cases
tests/uploads
# Don't commit ZIP files, except what's required for testing
*.7z
*.zip
!tests/*.zip
# Documents
*.doc*
*.pdf
# Avoid media files
*.avi
*.mp*
*.wmv
# Backup files
~$*
*~
*.bak*
# Sublime-text workspaces, etc
*.sublime-*
.vscode/
# IPython Notebook checkpoints
.ipynb_checkpoints
# Typically bash.exe.stackdump on Cygwin
*.stackdump
# Node modules and bower components
node_modules
bower_components
# Prefer yarn.lock over package-lock.json
package-lock.json
# Windows shortcut files
*.lnk
# Windows / Mac OS junk files
Desktop.ini
$RECYCLE.BIN/
*[Tt]humbs.db
*.DS_Store
# R history files
.RHistory
# C extensions
*.so
# Packages
*.egg
*.eggs
*.egg-info
dist
build
eggs
parts
bin
var
sdist
develop-eggs
.installed.cfg
lib
lib64
# Installer logs
pip-log.txt
# Unit test / coverage reports
.coverage
.tox
nosetests.xml
htmlcov
cover
# Translations
*.mo
# Mr Developer
.mr.developer.cfg
.project
.pydevproject
# Pycharm
.idea
# Complexity
output/*.html
output/*/index.html
# Sphinx
docs/_build
# For Linux FUSE file system
.fuse_hidden*
\ No newline at end of file
validate:
script:
- builderrors v2
\ No newline at end of file
FROM conda/miniconda3:latest
LABEL description="Gramener builderros for CI"
LABEL version="0.1"
LABEL maintainer="cto@gramener.com"
# Install nodejs and node libraries
RUN apt-get update && apt-get install -y gnupg curl wget git
RUN (curl -sL https://deb.nodesource.com/setup_10.x | bash -) && \
apt-get install -y nodejs && npm install -g yarn
# Install psycopg2 and psutil depenedencies
RUN apt-get install -y python3-dev build-essential
# validate_files does duplicate file checking using fdupes
RUN apt-get install -qq -y libpq-dev fdupes make
# Install validate dependencies
RUN pip install flake8 flake8-gramex flake8-blind-except flake8-print flake8-debugger pep8-naming
RUN pip install bandit
RUN npm install -g eslint eslint-plugin-template eslint-plugin-html \
stylelint@9 stylelint-config-recommended htmllint-cli \
eclint bower jscpd@0.6
# Install Gramex 1.x and dependencies
## Required for Gramex CaptureHandler tests
RUN pip install pycryptodome
## Required for Gramex CaptureHandler tests
RUN conda install -y -c conda-forge pdfminer.six
RUN pip install nose testfixtures coverage python-dateutil websocket-client sphinx_rtd_theme
## Required for gramex.ml
RUN conda install -y scikit-learn rpy2 tzlocal
RUN R -e "install.packages('backports', dependencies=TRUE, repos='https://cran.microsoft.com/')"
RUN R -e "install.packages('ggplot2', dependencies=TRUE, repos='https://cran.microsoft.com/')"
RUN R -e "install.packages('rmarkdown', dependencies=TRUE, repos='https://cran.microsoft.com/')"
# Install PhantomJS
RUN wget https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-2.1.1-linux-x86_64.tar.bz2 && \
tar -xvjf phantomjs-2.1.1-linux-x86_64.tar.bz2 && \
rm phantomjs-2.1.1-linux-x86_64.tar.bz2 && \
apt-get install -y libfontconfig && \
cd /usr/bin && \
ln -s /phantomjs-2.1.1-linux-x86_64/bin/phantomjs && \
cd /
# Install Chrome / Puppeteer dependencies
# https://github.com/GoogleChrome/puppeteer/issues/404#issuecomment-323555784
RUN apt-get install -y libpangocairo-1.0-0 libx11-xcb1 libxcomposite1 libxdamage1 libxi6 libxtst6 libnss3 libcups2 libxss1 libxrandr2 libgconf2-4 libasound2 libatk1.0-0 libgtk-3-0
# Install Selenium driver dependency: Chrome
RUN echo 'deb http://dl.google.com/linux/chrome/deb/ stable main' > /etc/apt/sources.list.d/google-chrome.list
RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -
RUN apt-get update && apt-get -y install google-chrome-stable
# Install Selenium Chromedriver
RUN apt-get -y install unzip
RUN wget -q https://chromedriver.storage.googleapis.com/2.37/chromedriver_linux64.zip
RUN unzip chromedriver_linux64.zip
RUN mv chromedriver /usr/bin/chromedriver
RUN chown root:root /usr/bin/chromedriver
RUN chmod +x /usr/bin/chromedriver
RUN rm chromedriver_linux64.zip
# Install Selenium-based test automation dependences
RUN pip install selenium pytest allure-pytest pytest-html
# Install load test dependencies
RUN apt-get install -y default-jre-headless libxml2-dev libxslt-dev zlib1g-dev net-tools default-jre default-jdk zip unzip
RUN pip install bzt
# Install dev version of gramex
RUN git clone https://github.com/gramener/gramex.git
RUN pip install -e gramex && gramex setup --all && gramex license accept
# Install builderrors
RUN git clone https://PratapVardhan@bitbucket.org/PratapVardhan/builderrors.git
RUN cd builderrors && yarn install && pip install pyminifier && chmod a+x builderrors && cd /
# create validate link
RUN ln -s /builderrors/builderrors /bin/validate
\ No newline at end of file
This diff is collapsed.
This diff is collapsed.
# Build error configurations
bandit-desc:
# Confidence - Severity: [description, error=true/false]
LOW-LOW: ['Low risk?', false]
LOW-MEDIUM: ['Danger?', false]
LOW-HIGH: ['Critical?', false]
MEDIUM-LOW: ['Low risk, maybe', false]
MEDIUM-MEDIUM: ['Danger, maybe', false]
MEDIUM-HIGH: ['Critical, maybe', false]
HIGH-LOW: ['Low risk', false]
HIGH-MEDIUM: ['Danger', true]
HIGH-HIGH: ['Critical', true]
jscpd-err: 'ERROR: {:,d} duplicate lines (max: {:,d}). https://goo.gl/UUiD3r#duplicate-lines'
charcount-err: '{}: {:,d} minified chars in {} files (max: {:,d}). https://goo.gl/UUiD3r#large-code'
flake8-warn: 'WARNING: %d complex Python functions. https://goo.gl/UUiD3r#complexity'
flake8-err: 'ERROR: %d Flake8 errors https://goo.gl/UUiD3r#python-lint'
eslint-warn: 'WARNING: %d complex JS functions. https://goo.gl/UUiD3r#complexity'
eslint-err: 'ERROR: %d eslint errors. https://goo.gl/UUiD3r#js-lint'
bandit-err: '%s: %d Python security errors. https://goo.gl/UUiD3r#security'
'''
Build error utilities
builderrorutils.py yamllint <dir> # Checks gramex.yaml at <dir>
builderrorutils.py charcount 500 1000 py,js . # Warn if over 500 PY/JS chars, Error over 1000
builderrorutils.py charcount 500 1000 css . # Warn if over 500 CSS chars, Error over 1000
flake8 . | builderrorutils.py flake8parse # Split flake8 errors and warnings
eslint -f unix| builderrorutils.py eslintparse # Split eslint errors and warnings
builderrorutils.py jscpd 100 report.json # Formats jscpd report if over 100 duplicates
'''
from __future__ import unicode_literals
from __future__ import print_function
import csv
import gramex.cache
import gramex.config
import io
import json
import yaml
import os
import six
import sys
try:
from pathlib2 import Path
except ImportError:
from pathlib import Path
from fnmatch import fnmatch
from subprocess import check_output, CalledProcessError
conf = gramex.cache.open('builderrors.yaml', rel=True, Loader=yaml.FullLoader)
def jscpd(count, path):
'''
If number of duplicates is more than count, print duplicate lines.
Else do not print anything.
'''
with io.open(path, encoding='utf-8') as handle:
jscpd = json.load(handle)
dups = jscpd['statistics']['duplications']
count = int(count)
if dups < count:
return
print(conf['jscpd-err'].format(dups, count))
max_files = 10
dups = sorted(jscpd['duplicates'], key=lambda v: v['lines'], reverse=True)
for dup in dups[:max_files]:
print(' - %d lines' % dup['lines'])
print(' %(name)s:%(start)d' % dup['firstFile'])
print(' %(name)s:%(start)d' % dup['secondFile'])
if len(dups) > max_files:
print(' ... %d more files' % (len(dups) - max_files))
print('')
sys.exit(1)
class ConfigStr(str):
'''subclass of string to store attributes'''
def __new__(cls, value, **kw):
if isinstance(value, six.text_type) and six.PY2:
value = value.encode('utf-8')
obj = str.__new__(cls, value)
obj.__dict__.update(kw)
return obj
def track_substitutes(f):
'''wrapper to return ConfigStr instance of f(x)'''
def wrapper(*args, **kwargs):
value = f(*args, **kwargs)
if isinstance(value, six.string_types):
return ConfigStr(value, __source__={'original': args[0]})
# if the value is not a string (e.g. dict), do not provide a __source__
return value
return wrapper
def yamllint(path='.', gramex_init=''):
# monkey-patch _substitute_variable with track_substitutes
gramex.config._substitute_variable = track_substitutes(gramex.config._substitute_variable)
gramex.config.Path = Path
config = gramex.config.PathConfig(os.path.join(os.path.abspath(path), 'gramex.yaml')) or {}
msg = []
# Deploy password mandatory if auth: is present
app_auth = config.get('app', {}).get('auth')
url_auth = any('Auth' in x.get('handler', '')
for x in config.get('url', {}).values())
if app_auth or url_auth:
test_section = config.get('test')
if test_section is None:
msg.append('test: required when using auth')
elif not test_section.get('auth'):
msg.append('test.auth: required when using auth')
# Ensure handlers use YAMLURL and YAMLPATH
for name, handler in config.get('url', {}).items():
if name == 'default':
continue
# Recommend $YAMLURL for url.*.pattern
if 'pattern' in handler:
pattern = handler.pattern
if '/$YAMLURL/' not in pattern.__source__['original']:
msg.append('url.{}: Use $YAMLURL in pattern:'.format(name))
# Recommend $YAMLPATH for FileHandler path for relative paths
if 'handler' in handler:
if 'kwargs' in handler and 'path' in handler.kwargs:
paths = handler.kwargs.path
paths = (
handler.kwargs.path.values() if isinstance(paths, dict) else
paths if isinstance(paths, list) else [paths])
for itempath in paths:
original = itempath.__source__['original']
p = os.path.normpath(itempath)
rel_path = os.path.abspath(p) != p
# If we use a relative path on a handler, use $YAMLPATH (or any variable)
if (handler.handler == 'FileHandler' and rel_path and '$' not in original):
msg.append('url.{}: Use $YAMLPATH in FileHandler path:'.format(name))
# Deprecate DataHandler/QueryHandler if gramex init was used
if (gramex_init and handler.handler in ['DataHandler', 'QueryHandler']):
msg.append('url.{}: Use FormHandler not Data/QueryHandler'.format(name))
for line in msg:
print(' %s' % line)
sys.exit(1 if len(msg) else 0)
node_bin = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'node_modules', '.bin')
minifiers = {
'css': [os.path.join(node_bin, 'cleancss'), '-O2', '--inline', 'none', '{}'],
'js': [os.path.join(node_bin, 'terser'), '{}', '--compress', '--mangle'],
'py': ['pyminifier', '--obfuscate', '{}'],
}
skip_patterns = ['*/node_modules/*']
def skip(path):
for pattern in skip_patterns:
if fnmatch(path, pattern):
return True
return False
def charcount(warncount, errorcount, extensions, path='.'):
extensions = set('.' + ext for ext in extensions.split(','))
shell = 'win' in sys.platform
counts = {}
for dirpath, dirnames, filenames in os.walk(path):
for filename in filenames:
target = os.path.join(dirpath, filename)
if skip(target):
continue
ext = os.path.splitext(filename.lower())[-1]
if ext in extensions:
minifier = [part.format(target) for part in minifiers[ext[1:]]]
try:
result = check_output(minifier, shell=shell) # nosec
except CalledProcessError:
print('WARNING: could not minify %s. Mail s.anand@gramener.com' % target)
with io.open(target, encoding='utf-8') as handle:
counts[target] = len(handle.read())
else:
counts[target] = len(result)
chars = sum(counts.values())
warncount, errorcount = int(warncount), int(errorcount)
has_error, max_files = False, 5
for errortype, count in (('ERROR', errorcount), ('WARNING', warncount)):
if chars > count:
print(conf['charcount-err'].format(errortype, chars, ','.join(extensions), count))
top_files = list(sorted(counts.items(), key=lambda v: v[1], reverse=True))
for name, filecount in top_files[:max_files]:
if filecount > 0:
print('{:10,d} {}'.format(filecount, name))
if len(top_files) > max_files:
print(' ... %d more files' % (len(top_files) - max_files))
print('')
has_error = errortype == 'ERROR' # Set error flag only if there's an error
break
sys.exit(1 if has_error else 0)
def split_error_warning(warning_match, warning, error_match, error):
'''Splits warning lines and error lines. Print them separately. Set error code if errors'''
errors, warnings = [], []
for line in sys.stdin:
if warning_match in line:
warnings.append(' ' + line)
elif error_match in line:
errors.append(' ' + line)
if errors:
print(error % len(errors))
print(''.join(errors))
if warnings:
print(warning % len(warnings))
print(''.join(warnings))
sys.exit(1 if errors else 0)
def flake8parse():
split_error_warning(
warning_match=': C901 ', warning=conf['flake8-warn'],
error_match='', error=conf['flake8-err'],
)
def eslintparse():
split_error_warning(
warning_match='[Warning/', warning=conf['eslint-warn'],
error_match='[Error/', error=conf['eslint-err'],
)
def banditparse(validate_version):
errors, warnings = [], []
for row in csv.DictReader(sys.stdin):
key = row['issue_confidence'] + '-' + row['issue_severity']
row['desc'], err = conf['bandit-desc'][key]
msg = (' {filename},{line_number}: {test_id} {issue_text} {desc}\n').format(**row)
(errors if err else warnings).append(msg)
for txt, e in (('ERROR', errors), ('WARNING', warnings)):
if e:
print(conf['bandit-err'] % (txt, len(e)))
print(''.join(e))
sys.exit(0 if validate_version < 'v2' else 1 if errors else 0)
if __name__ == '__main__':
if len(sys.argv) > 1:
globals()[sys.argv[1]](*sys.argv[2:])
else:
print(__doc__)
docker pull conda/miniconda3:latest
docker run --name runner -i -t conda/miniconda3:latest /bin/bash
## Inside docker
# Install nodejs and node libraries
apt-get update && apt-get install -y gnupg curl wget git
(curl -sL https://deb.nodesource.com/setup_10.x | bash -)
apt-get install -y nodejs && npm install -g yarn
# Install psycopg2 and psutil depenedencies
apt-get install -y python3-dev build-essential
# validate_files does duplicate file checking using fdupes
# make is also required
apt-get install -qq -y libpq-dev fdupes make
# Install validate dependencies
pip install flake8 flake8-gramex flake8-blind-except flake8-print flake8-debugger pep8-naming
pip install bandit
npm install -g eslint eslint-plugin-template eslint-plugin-html
npm install -g stylelint@9 stylelint-config-recommended htmllint-cli
npm install -g eclint bower jscpd@0.6
# Install Gramex 1.x and dependencies
pip install pycryptodome # Required for Gramex CaptureHandler tests
conda install -y -c conda-forge pdfminer.six # Required for Gramex CaptureHandler tests
pip install nose testfixtures coverage python-dateutil websocket-client sphinx_rtd_theme
conda install -y scikit-learn # ModelHandler Requires scikit-learn
conda install -y rpy2 tzlocal # Required for gramex.ml.r()
R --vanilla <<EOF
install.packages(c('backports', 'ggplot2', 'rmarkdown'), repos='https://cran.microsoft.com/')
EOF
# pip install -U tornado==4.5.3 # TODO: Gramex is not tested for tornado 5.x
# Install PhantomJS
wget https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-2.1.1-linux-x86_64.tar.bz2
tar -xvjf phantomjs-2.1.1-linux-x86_64.tar.bz2
rm phantomjs-2.1.1-linux-x86_64.tar.bz2
apt-get install -y libfontconfig
cd /usr/bin
ln -s /phantomjs-2.1.1-linux-x86_64/bin/phantomjs
cd /
# Install Chrome / Puppeteer dependencies
# https://github.com/GoogleChrome/puppeteer/issues/404#issuecomment-323555784
apt-get install -y libpangocairo-1.0-0 libx11-xcb1 libxcomposite1 libxdamage1 libxi6 libxtst6 libnss3 libcups2 libxss1 libxrandr2 libgconf2-4 libasound2 libatk1.0-0 libgtk-3-0
# Install Selenium driver dependency: Chrome
echo 'deb http://dl.google.com/linux/chrome/deb/ stable main' > /etc/apt/sources.list.d/google-chrome.list
wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -
apt-get update
apt-get -y install google-chrome-stable
# Install Selenium Chromedriver
apt-get -y install unzip
wget -q https://chromedriver.storage.googleapis.com/2.37/chromedriver_linux64.zip
unzip chromedriver_linux64.zip
mv chromedriver /usr/bin/chromedriver
chown root:root /usr/bin/chromedriver
chmod +x /usr/bin/chromedriver
rm chromedriver_linux64.zip
# Install Selenium-based test automation dependences
pip install selenium pytest allure-pytest pytest-html
# Install load test dependencies
apt-get install -y default-jre-headless libxml2-dev libxslt-dev zlib1g-dev net-tools default-jre default-jdk zip unzip
pip install bzt
# Install dev version of gramex
git clone https://github.com/gramener/gramex.git
pip install -e gramex && gramex setup --all && gramex license accept
# Install builderrors
git clone https://PratapVardhan@bitbucket.org/PratapVardhan/builderrors.git
cd builderrors && yarn install && pip install pyminifier && chmod a+x builderrors && cd /
# create validate link
ln -s /builderrors/builderrors /bin/validate
exit
## Outside docker
export VERSION=1.x
docker commit -m "Create validator image" -a "pv" runner pratapvardhan/validator:$VERSION
docker tag pratapvardhan/validator:$VERSION pratapvardhan/validator:latest
# docker rm runner
docker push pratapvardhan/validator
\ No newline at end of file
{
"dependencies": {
"bootstrap": "3.3.7",
"clean-css-cli": "4",
"stupid-table-plugin": "^1.1.3",
"terser": "3",
"underscore": "^1.9.1"
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment