Commit 979865d5 authored by Pratap Vardhan's avatar Pratap Vardhan

ENH: builderrors

parents
Pipeline #92129 failed with stage
in 24 seconds
TODO
# Ignore auto-generated documentation
docs/gramex*.rst
docs/modules.rst
# Ignore files generated by testcases
tests/**/*.test
tests/**/gen.*
testlib/**/gen.*
# Ignore byte-compiled / optimised / DLL files
*.py[cod]
# Filenames should NOT have spaces
* *
# Ignore SQLite3 files. Gramex creates some automatically
*.sqlite3
*.sqlite3-journal
# Cache folders used for testing
.cache-*
.pytest_cache
# Ignore log files
*.log*
# Don't commit data files, except what's required for testing or by Gramex apps
*.csv
*.xls*
!tests/**/*.csv
!tests/*.xlsx
!testlib/**/*.csv
!testlib/*.xlsx
!gramex/apps/**/*.csv
!gramex/apps/guide/**/*.xlsx
*.ppt*
!testlib/input.pptx
!tests/template.pptx
!gramex/apps/guide/formhandler/input.pptx
!gramex/apps/guide/pptxhandler/examples-input.pptx
# Don't commit databases created by test cases
tests/*.db
# Don't commit uploads created by test cases
tests/uploads
# Don't commit ZIP files, except what's required for testing
*.7z
*.zip
!tests/*.zip
# Documents
*.doc*
*.pdf
# Avoid media files
*.avi
*.mp*
*.wmv
# Backup files
~$*
*~
*.bak*
# Sublime-text workspaces, etc
*.sublime-*
.vscode/
# IPython Notebook checkpoints
.ipynb_checkpoints
# Typically bash.exe.stackdump on Cygwin
*.stackdump
# Node modules and bower components
node_modules
bower_components
# Prefer yarn.lock over package-lock.json
package-lock.json
# Windows shortcut files
*.lnk
# Windows / Mac OS junk files
Desktop.ini
$RECYCLE.BIN/
*[Tt]humbs.db
*.DS_Store
# R history files
.RHistory
# C extensions
*.so
# Packages
*.egg
*.eggs
*.egg-info
dist
build
eggs
parts
bin
var
sdist
develop-eggs
.installed.cfg
lib
lib64
# Installer logs
pip-log.txt
# Unit test / coverage reports
.coverage
.tox
nosetests.xml
htmlcov
cover
# Translations
*.mo
# Mr Developer
.mr.developer.cfg
.project
.pydevproject
# Pycharm
.idea
# Complexity
output/*.html
output/*/index.html
# Sphinx
docs/_build
# For Linux FUSE file system
.fuse_hidden*
\ No newline at end of file
validate:
script:
- builderrors v2
\ No newline at end of file
FROM conda/miniconda3:latest
LABEL description="Gramener builderros for CI"
LABEL version="0.1"
LABEL maintainer="cto@gramener.com"
# Install nodejs and node libraries
RUN apt-get update && apt-get install -y gnupg curl wget git
RUN (curl -sL https://deb.nodesource.com/setup_10.x | bash -) && \
apt-get install -y nodejs && npm install -g yarn
# Install psycopg2 and psutil depenedencies
RUN apt-get install -y python3-dev build-essential
# validate_files does duplicate file checking using fdupes
RUN apt-get install -qq -y libpq-dev fdupes make
# Install validate dependencies
RUN pip install flake8 flake8-gramex flake8-blind-except flake8-print flake8-debugger pep8-naming
RUN pip install bandit
RUN npm install -g eslint eslint-plugin-template eslint-plugin-html \
stylelint@9 stylelint-config-recommended htmllint-cli \
eclint bower jscpd@0.6
# Install Gramex 1.x and dependencies
## Required for Gramex CaptureHandler tests
RUN pip install pycryptodome
## Required for Gramex CaptureHandler tests
RUN conda install -y -c conda-forge pdfminer.six
RUN pip install nose testfixtures coverage python-dateutil websocket-client sphinx_rtd_theme
## Required for gramex.ml
RUN conda install -y scikit-learn rpy2 tzlocal
RUN R -e "install.packages('backports', dependencies=TRUE, repos='https://cran.microsoft.com/')"
RUN R -e "install.packages('ggplot2', dependencies=TRUE, repos='https://cran.microsoft.com/')"
RUN R -e "install.packages('rmarkdown', dependencies=TRUE, repos='https://cran.microsoft.com/')"
# Install PhantomJS
RUN wget https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-2.1.1-linux-x86_64.tar.bz2 && \
tar -xvjf phantomjs-2.1.1-linux-x86_64.tar.bz2 && \
rm phantomjs-2.1.1-linux-x86_64.tar.bz2 && \
apt-get install -y libfontconfig && \
cd /usr/bin && \
ln -s /phantomjs-2.1.1-linux-x86_64/bin/phantomjs && \
cd /
# Install Chrome / Puppeteer dependencies
# https://github.com/GoogleChrome/puppeteer/issues/404#issuecomment-323555784
RUN apt-get install -y libpangocairo-1.0-0 libx11-xcb1 libxcomposite1 libxdamage1 libxi6 libxtst6 libnss3 libcups2 libxss1 libxrandr2 libgconf2-4 libasound2 libatk1.0-0 libgtk-3-0
# Install Selenium driver dependency: Chrome
RUN echo 'deb http://dl.google.com/linux/chrome/deb/ stable main' > /etc/apt/sources.list.d/google-chrome.list
RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -
RUN apt-get update && apt-get -y install google-chrome-stable
# Install Selenium Chromedriver
RUN apt-get -y install unzip
RUN wget -q https://chromedriver.storage.googleapis.com/2.37/chromedriver_linux64.zip
RUN unzip chromedriver_linux64.zip
RUN mv chromedriver /usr/bin/chromedriver
RUN chown root:root /usr/bin/chromedriver
RUN chmod +x /usr/bin/chromedriver
RUN rm chromedriver_linux64.zip
# Install Selenium-based test automation dependences
RUN pip install selenium pytest allure-pytest pytest-html
# Install load test dependencies
RUN apt-get install -y default-jre-headless libxml2-dev libxslt-dev zlib1g-dev net-tools default-jre default-jdk zip unzip
RUN pip install bzt
# Install dev version of gramex
RUN git clone https://github.com/gramener/gramex.git
RUN pip install -e gramex && gramex setup --all && gramex license accept
# Install builderrors
RUN git clone https://PratapVardhan@bitbucket.org/PratapVardhan/builderrors.git
RUN cd builderrors && yarn install && pip install pyminifier && chmod a+x builderrors && cd /
# create validate link
RUN ln -s /builderrors/builderrors /bin/validate
\ No newline at end of file
We run automated checks on repositories to improve code quality.
This page lists the checks we run, what they do, and how to fix them.
# Usage
The best way to run checks is to commit a `.gitlab-ci.yml` file that has this text:
```yaml
validate: # Standard name of validation process
script: validate v2 # Check for Gramex 1.x errors
```
- For a less strict build error check, use `script: validate v1` instead
When you commit to <https://code.gramener.com/>, it automatically runs these
checks and mails you if any critical check failed. Visit your project on
<https://code.gramener.com/>, go to "Pipelines" to see the detailed results.
## Local usage
To run these checks locally, install [Anaconda](http://continuum.io/downloads),
[node.js](https://nodejs.org/en/) and `git`. Then set these packages up:
```bash
pip install gramex
pip install flake8 flake8-gramex flake8-blind-except flake8-print flake8-debugger pep8-naming
pip install bandit
npm install -g eslint eslint-plugin-template eslint-plugin-html
npm install -g stylelint@9
npm install -g htmllint-cli
npm install -g jscpd@0.6
sudo apt-get install fdupes # On Windows, use Cygwin to install fdupes
# Clone the repo
git clone https://code.gramener.com/cto/monitor.git
```
Now you can run `monitor/builderrors` and see the results.
# Checks
The [builderrors](builderrors) script runs a series of checks. Here's what each error means, and how to fix it.
## Libraries
`ERROR: don't commit libraries`
**Explanation**: Don't commit the `bower_components` or `node_modules`
directories. Install them in the target system using [Gitlab CI][ci] via `yarn
install`.
**Solution**:
- add `bower_components/` and `node_modules/` to your `.gitignore`
- run `git rm -rf node_modules/ bower_components/` from your repo root
## g1 library
- v2: `ERROR: Upgrade to G1 Library`
- v1: `WARNING: Upgrade to G1 Library`
**Explanation**: The [G library](https://learn.gramener.com/docs/g) included via
`G.min.js` is deprecated. Use the [g1 library](https://code.gramener.com/cto/g1/).
**Solution**:
- Run `yarn install g1` to install the g1 library
- Include `<script src="node_modules/g1/dist/g1.min.js"></script>` in your code
- Remove `G.min.js` from your source, and from `bower.json` if present
## Minified files
`ERROR: don't commit minified files`
**Explanation**: Don't commit minified files like `jquery.min.js`,
`bootstrap.min.css`, etc. Install them in the target system using [Gitla CI][ci]
via `yarn install`.
**Solution**:
- If the minified files are part of an npm library:
- run `git rm <minified-file`
- run `yarn add <lib>`
- change links to point to the new location (e.g. `node_modules/<lib>/dist/<lib>.min.js`)
- If the minified files are *not* part of an npm library:
- run `git rm <file.min.css>`
- run `git add <file.css>` -- commit the un-minified file
- run `yarn add uglify-js`
- in `package.json`, add a script `minify: "uglifyjs file.css -o file.min.css"`
- in `setup.sh`, add `yarn run minify`
## Large files
`ERROR: don't commit files over 1MB`
**Explanation**: Large are not easy to maintain on git. Copy them from on install.
**Solution**:
When deploying to uat.gramener.com, copy the file to `demo.gramener.com` under `/mnt/deploy/` using
`rsync -avzP path/to/large.file <your-id>@demo.gramener.com:/deploy/<user>/<repo>/path/to/large.file`.
This directory will be automatically copied when deploying on `uat.gramener.com`.
The `/<user>/<repo>` must be the same as the path to your repo on `https://code.gramener.com/<user>/<repo>`
## Binary files
`ERROR: don't commit binary files`
**Explanation**: Binary files are not easy to maintain on git. Copy them from on install.
**Solution**:
When deploying to uat.gramener.com, copy the file to `demo.gramener.com` under `/mnt/deploy/` using
`rsync -avzP path/to/large.file <your-id>@demo.gramener.com:/deploy/<user>/<repo>/path/to/large.file`.
This directory will be automatically copied when deploying on `uat.gramener.com`.
The `/<user>/<repo>` must be the same as the path to your repo on `https://code.gramener.com/<user>/<repo>`
## Useless files
`ERROR: don't commit useless files`
**Explanation**: Files that are not needed to run the app should not be
committed. For example, log files (`*.log`), thumbnail databases (`thumbs.db`),
backup files (`*~`), etc.
**Solution**:
- add `<useless.file>` to your `.gitignore`
- run `git rm <useless.file>`
## Duplicate files
`ERROR: don't duplicate files`
**Explanation**: The same non-empty file is present twice in the same repo.
**Solution**:
- run `git rm <duplicate.file>`
- change code to point the deleted `<duplicate.file>` to the retained file.
## Complexity
- `WARNING: xx complex Python functions`
- `WARNING: xx complex JS functions`
**Explanation**: Long or complex functions are difficult to understand. Break it
into smaller functions.
**Solution**:
- Think how the function can be broken into parts that can be used by other functions
- Refactor the code and test *very carefully*. (Unit test cases help here.)
## Gramex init
- v2: `ERROR: Use gramex init to start projects`
- v1: `WARNING: Use gramex init to start projects`
**Explanation**: `gramex init` is the recommended way to initiate a new Gramex
project. It automatically installs the required files.
**Solution**:
- Move your `gramex.yaml` file
- Run `gramex init` in your repo
- Copy-paste the contents of your `gramex.yaml` into the new `gramex.yaml` -- but retain the first comment line that says "Generated by gramex init..."
## Gramex YAML
`ERROR: gramex.yaml issues`
**Explanation**: The `gramex.yaml` file has some commonly made mistakes. Fix these
**Solution**:
- `test: required when using auth` or `test.auth: required when using auth`:
This is to help the Gramener team log into deployments at <https://uat.gramener.com/monitor/apps>.
If your app uses authentication, add a test user credentials to `gramex.yaml` like this:
```yaml
test:
auth:
user: test-user-name
password: test-user-password # optional, suggested only for SimpleAuth / DBAuth
```
To specify multiple usernames, make the `test.auth:` section a list:
```yaml
test:
auth:
- {user: test-user-name, password: tets-user-password}
- {user: next-user-name, password: next-user-password}
- ...
```
- `url.<key>: Use $YAMLURL in pattern:`: URL `pattern:` should normally begins with `/$YAMLURL/`.
- `url.<key>: Use $YAMLPATH in FileHandler path:`: Relative `path:` in FileHandler should normally begin with `$YAMLPATH/`.
- `url.<key>: Use FormHandler not Data/QueryHandler`: [DataHandler][datahandler] and [QueryHandler][queryhandler] are deprecated
## YAML length
`ERROR: keep YAML lines under 120 chars`
**Explanation**: Long YAML lines are not readable. Keep them under 120 characters.
**Solution**:
If the line has spaces, just quote the value and break at the spaces. For example:
```yaml
key: "This sentence is split into 2 lines.
Leading whitespace is ignored."
# Same as "This sentence is split into 2 lines. Leading whitespace is ignored."
```
To avoid adding spaces, `\` at the end of the line. For example:
```yaml
key: "https://some-long-url/abc?\
key1=value1&\
key2=value2"
# Same as "https://some-long-url/abc?key1=value1&key2=value2"
```
See [yaml-multiline.info](http://yaml-multiline.info/) for more options
## Python files
`ERROR: Python paths must be lower_alphanumeric`
**Explanation**: Python filenames must be valid identifiers to import them as modules. The convention is to use lowercase alphanumerics.
**Solution**:
Rename the Python files, ensuring that:
- the filename only uses lower case letters, numbers and underscore (`_`)
- the filename starts with a letter
## Large code
- v2: `ERROR: 20,000 chars in .css files (max: 10,000)`
- v2: `ERROR: 80,000 chars in .py, .js files (max: 50,000)`
- v1: `WARNING: 20,000 chars in .css files (max: 10,000)`
- v1: `WARNING: 80,000 chars in .py, .js files (max: 50,000)`
**Explanation**: The project has too much code. Projects should have a small code base, and re-use code from libraries.
**Solution**:
- Use [Gramex][gramex] features to reduce the code size
- Use third party libraries
- Contact cto@gramener.com to add functionality into [Gramex][gramex]
## Duplicate lines
- v1: `ERROR: 1,200 duplicate lines (max: 1,000)`
- v2: `ERROR: 1,200 duplicate lines (max: 750)`
**Explanation**: The project repeats too much code. Duplicate code should be re-factored into functions.
**Solution**:
- Use **loops** for code repeated one after another
- Use **functions** for code repeated in different places (either in the same file or different files)
- Use **function parameters** to handle minor variations in the repetition
- Use **data structures** for larger variations. For example, create an array or
dictionary that stores all the parameters that vary. Remember: you can use functions as values.
- Use **function generators** for extreme variations in code. Write a function
to create and return a new function depending on your variation.
- Refactor the code and test *very carefully*. (Unit test cases help here.)
## editorconfig
`ERROR: follow .editorconfig rules`
**Explanation**: [.editorconfig](https://editorconfig.org) defines conventions
for editor indentation. Projects must follow these.
**Solution**:
- If your repo does not have an `.editorconfig`, run `gramex init` to create it (one-time)
- run `npm install -g eclint` to install eclint (one-time)
- run `eclint fix <file>` to fix most errors in `<file>`
- run `eclint check <file>` to see remaining errors and fix them manually
## Python lint
`ERROR: 10 Flake8 errors`
**Explanation**: [flake8][flake8] checks Python code and reports errors.
**Solution**:
[autopep8][autopep8] auto-fixes many flake8 errors. e.g. [VariantAnalysis.py][sample-code]
reports **293 errors**. autopep8 reduces that to **12 errors** automatically.
Run:
pip install autopep8
autopep8 -iv --max-line-length 99 *.py
These are the most common errors that it **does not** fix are below. (These also
indicate **possible errors** in your code!)
- **W191**: indentation contains tabs. **Replace tabs with spaces**. e.g. on [SublimeText](https://stackoverflow.com/a/9575774/100904).
- **E910**: Specify encoding= for ... **Add an `encoding='utf-8'`** when using `io.open` or `pd.read_*`. Or use [gramex.cache.open](https://learn.gramener.com/guide/cache/#data-caching).
- **E911**: Use `.format` or `%` instead of `str()`. `str()` is dangerous. `str(0.3 - 1e-13)` returns different values on Python 2 vs Python 3. It even returns different *types* (unicode vs bytes.)
- **E912**: Define magic constant ... as a variable. For example, `x * 86400` isn't as clear as `x * seconds_per_day`. Convert constants into clearly named variables.
- **F841**: local variable '...' is assigned to but never used. **Check if you forgot**, else don't assign it
- **F401**: '...' imported but unused. **Check if you forgot** to use the module. Else don't import it
- **F821**: undefined name ... You used an uninitialized variable. That's wrong.
- **F811**: redefinition of unused '...'. You assigned a variable and never used it. Then you're reassigning it. Or re-importing. Look carefully.
- **B901**: blind except: statement. Trap **specific** exceptions. Blind exceptions can trap even syntax errors and confuse you later.
- **F601**: dictionary key '...' repeated with different values. e.g. `{'x': 1, 'x': 2}`. That's wrong.
Here are a few other common errors that you want to avoid.
- **T001** or **T003**: print found -- just remove `print` in production code.
- **N806**: variable in function should be lowercase -- rename your variable.
- **N802** or **N803**: function and argument names should be lowercase.
The default flake8 rules are from `gramex init`. You can create / change your
repository's `.flake8` to add/ignore specific rules.
## CSS lint
- v2: `ERROR: stylelint errors (CSS)`
- v1: `WARNING: stylelint errors (CSS)`
**Explanation**: [stylelint][stylelint] checks CSS and reports errors.
**Solution**: Re-write the code based on advice from [stylelint][stylelint]
## JS lint
`ERROR: 20 eslint errors`
**Explanation**: [eslint][eslint] checks JS and reports errors.
**Solution**:
- run `eslint --fix` to automatically fix eslint errors where possible.
- run `eslint --ext js,html .` to review errors manually and fix the rest. Visit <http://eslint.org/docs/rules/> to see all rules.
Here are some common errors:
- [no-undef](http://eslint.org/docs/rules/no-undef): you are using a variable not
defined in the file. Begin your JS with `/* globals var1, var2, ... */` for
each undefined `var1`, `var2`, etc.
- [no-unused-vars](http://eslint.org/docs/rules/no-unused-vars): you are
assigning to a variable or defining a function, but not using it in the file.
If you're exporting it, begin your JS with `/* exported var1, fn1, ... */`.
Else delete / don't assign.
## HTML lint
`ERROR: html errors`
**Explanation**: [htmllint][htmllint] checks HTML and reports errors.
**Solution**: Re-write the code based on advice from [htmllint][htmllint]
## Security
- v2: `ERROR: 3 Python security errors`
- v1: `WARNING: 3 Python security errors`
**Explanation**: [bandit][bandit] shows security errors in Python code. Fix them.
**Solution**: Re-write the code based on advice from [bandit][bandit].
## Environment variables
Build errors behavior is controlled using
[Gitlab CI environment variables](https://docs.gitlab.com/ce/ci/variables/)
under your project > Settings > CI / CD > Variables. Set these variables and
re-run the pipeline to see the effect.
- CSS_CHARS_WARN=6000: if minified CSS exceeds 6,000, warn about [large code](#large-code)
- CSS_CHARS_ERROR=10000: if minified CSS exceeds 10,000, raise error about [large code](#large-code)
- CODE_CHARS_WARN=30000: if minified JS+PY exceeds 30,000, warn about [large code](#large-code)
- CODE_CHARS_ERROR=50000: if minified JS+PY exceeds 50,000, raise error about [large code](#large-code)
[ci]: https://docs.gitlab.com/ce/ci/yaml/
[datahandler]: https://learn.gramener.com/guide/datahandler/
[queryhandler]: https://learn.gramener.com/guide/queryhandler/
[gramex]: https://learn.gramener.com/guide/
[stylelint]: https://github.com/stylelint/stylelint
[bandit]: https://github.com/openstack/bandit
[flake8]: http://flake8.pycqa.org/
[autopep8]: https://pypi.python.org/pypi/autopep8
[sample-code]: https://code.gramener.com/group-m/Mindshare-HUL/blob/b5eb09dda0912436eea4a489468cd2d4e50bed34/VariantAnalysis.py
[eslint]: http://eslint.org/
[htmllint]: https://github.com/htmllint/htmllint/wiki
#!/usr/bin/env bash
# This script runs in these modes, based on what the script name is:
# validate_files: Checks file errors only
# validate: Also checks Gramex 0.x flake8, eslint and htmllint
# validate v1: Also checks Gramex 1.x flake8, eslint and htmllint
# validate v2: Runs stricter checks for 2019 repos
# builderrors is an alias for validate
# Check these errors via https://code.gramener.com/s.anand/validatecheck
# Keep this consistent with http://learn.gramener.com/wiki/gitlab.html
REPO_DIR="`pwd`"
SCRIPT_NAME=`basename "$0"`
# Use v0 validation by default, unless the first command line parameter specifies it
# Running as "builderrors" is the same as "validate v1"
GRAMEX_VALIDATE="v0"
if [ "$SCRIPT_NAME" == "builderrors" ]; then