From 5fa3d48859bd4c5ddccd94e10513b9ab8b9d1300 Mon Sep 17 00:00:00 2001 From: Simon Burns Date: Wed, 9 Dec 2015 13:11:44 +0000 Subject: [PATCH] Initial commit, with files --- Procfile | 50 +++++++++++++ README.md | 32 ++++++++ Vagrantfile | 17 +++++ env | 201 +++++++++++++++++++++++++++++++++++++++++++++++++++ huginn | 70 ++++++++++++++++++ provision.sh | 112 ++++++++++++++++++++++++++++ 6 files changed, 482 insertions(+) create mode 100644 Procfile create mode 100644 README.md create mode 100644 Vagrantfile create mode 100644 env create mode 100644 huginn create mode 100755 provision.sh diff --git a/Procfile b/Procfile new file mode 100644 index 0000000..8c83b82 --- /dev/null +++ b/Procfile @@ -0,0 +1,50 @@ +############################### +# DEVELOPMENT # +############################### + +# Procfile for development using the new threaded worker (scheduler, twitter stream and delayed job) +web: bundle exec rails server -p ${PORT-3000} -b ${IP-0.0.0.0} +jobs: bundle exec rails runner bin/threaded.rb + +# Old version with separate processes (use this if you have issues with the threaded version) +# web: bundle exec rails server +# schedule: bundle exec rails runner bin/schedule.rb +# twitter: bundle exec rails runner bin/twitter_stream.rb +# dj: bundle exec script/delayed_job run + +############################### +# PRODUCTION # +############################### + +# You need to copy or link config/unicorn.rb.example to config/unicorn.rb for both production versions. +# Have a look at the deployment guides, if you want to set up huginn on your server: +# https://github.com/cantino/huginn/doc + +# Using the threaded worker (consumes less RAM but can run slower) +web: bundle exec unicorn -c config/unicorn.rb +jobs: bundle exec rails runner bin/threaded.rb + +# Old version with separate processes (use this if you have issues with the threaded version) +# web: bundle exec unicorn -c config/unicorn.rb +# schedule: bundle exec rails runner bin/schedule.rb +# twitter: bundle exec rails runner bin/twitter_stream.rb +# dj: bundle exec script/delayed_job run + +############################### +# Multiple DelayedJob workers # +############################### +# Per default Huginn can just run one agent at a time. Using a lot of agents or calling slow +# external services frequently might require more DelayedJob workers (an indicator for this is +# a backlog in your 'Job Management' page). +# Every uncommented line starts an additional DelayedJob worker. This works for development, production +# and for the threaded and separate worker processes. Keep in mind one worker needs about 300MB of RAM. +# +#dj2: bundle exec script/delayed_job -i 2 run +#dj3: bundle exec script/delayed_job -i 3 run +#dj4: bundle exec script/delayed_job -i 4 run +#dj5: bundle exec script/delayed_job -i 5 run +#dj6: bundle exec script/delayed_job -i 6 run +#dj7: bundle exec script/delayed_job -i 7 run +#dj8: bundle exec script/delayed_job -i 8 run +#dj9: bundle exec script/delayed_job -i 9 run +#dj10: bundle exec script/delayed_job -i 10 run diff --git a/README.md b/README.md new file mode 100644 index 0000000..4a0229a --- /dev/null +++ b/README.md @@ -0,0 +1,32 @@ +# Huginn with Vagrant and VirtualBox + +Based on instructions here: + +https://github.com/cantino/huginn/blob/master/doc/manual/installation.md + +I created this installation for Vagrant/VirtualBox because I couldn't get email +working from the Docker image. + +- Clone this repository: `git clone https://github.com/m0nty/huginn-vagrant.git` +- Edit the `env` file, which will be copied to `/home/huginn/huginn/.env` + (look for the FIXME comments in the file to see what you need to adjust). +- Edit `provision.sh` and change the MySQL passwords for the root MySQL user and + the Huginn DB user (search for FIXME again). +- Edit the `Vagrantfile` if you want to change the virtual machine settings. I've + gone with `vb.memory = "1024"` which you might want to increase if you think + your Huginn needs more. +- You might want to change this line `config.vm.box = "ubuntu/trusty64"` if your + PC can't run 64-bit images. +- Edit the `huginn` file if you want to change the listening address and port for + your Huginn instance. +- I don't think you need to change the Procfile, but you may as well check anyway. + +Please let me know if I've messed up in any way. I've got my Huginn instance working +now, and I'm delighted with it. I just wanted to help out if anyone else is having +problems with the other installation methods. + +# To-do + +I'll probably do a Dockerfile sometime, so you can create a docker instance yourself +without pulling the Huginn docker image. + diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 0000000..48ac87e --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,17 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +Vagrant.configure(2) do |config| + config.vm.box = "ubuntu/trusty64" + config.vm.define "huginn-vagrant" do |huginn| + end + config.vm.provider "virtualbox" do |vb| + vb.gui = false + vb.memory = "1024" + vb.name = "huginn-vagrant" + end + config.vm.hostname = "huginn.local" + config.vm.box_check_update = false + config.vm.provision "shell", inline: "/vagrant/provision.sh" + config.vm.network :forwarded_port, guest:3000, host:3000, id:"nginx", host_ip:"127.0.0.1", auto_correct: false +end diff --git a/env b/env new file mode 100644 index 0000000..2cbee8d --- /dev/null +++ b/env @@ -0,0 +1,201 @@ +# ==== Required configuration settings for Huginn ==== + +# Replace the following with the output from "rake secret" +# FIXME - next line +APP_SECRET_TOKEN=4714932e88290ad2bacbafc56a8d1df59ca02e3fde1eaec379d96a2834e1d9cc15a80610d432246af74a9cef1c0974e0acdff08fac44a2f7c8cdb3a230e212bd + +# This is the domain where your Huginn instance will be running. The default should work +# for development, but it needs to be changed to your Huginn domain when you deploy to a +# production environment (e.g., yourdomain.com, possibly including a port). +# FIXME - next line +DOMAIN=localhost:3000 + +# Optionally set an asset host +# ASSET_HOST=http://assets.example.com + +############################ +# Database Setup # +############################ + +# FIXME - At least the DATABASE_USERNAME and DATABASE_PASSWORD +DATABASE_ADAPTER=mysql2 +DATABASE_ENCODING=utf8mb4 +DATABASE_RECONNECT=true +DATABASE_NAME=huginn_production +DATABASE_POOL=20 +DATABASE_USERNAME=huginn +DATABASE_PASSWORD=mysqlpassword +#DATABASE_HOST=your-domain-here.com +#DATABASE_PORT=3306 +#DATABASE_SOCKET=/tmp/mysql.sock + +# MySQL only: If you are running a MySQL server >=5.5.3, you should +# set DATABASE_ENCODING to utf8mb4 instead of utf8 so that the +# database can hold 4-byte UTF-8 characters like emoji. +#DATABASE_ENCODING=utf8mb4 + +# ==== Additional required production settings ==== + +# Configure Rails environment. This should only be needed in production and may cause errors in development. +RAILS_ENV=production + +# Should Rails force all requests to use SSL? +FORCE_SSL=false + +############################ +# Allowing Signups # +############################ + +# This invitation code will be required for users to signup with your Huginn installation. +# You can see its use in user.rb. PLEASE CHANGE THIS! +INVITATION_CODE=try-huginn + +# If you don't want to require new users to have an invitation code in order to sign up, set this to true. +SKIP_INVITATION_CODE=false + +############################# +# Email Configuration # +############################# + +# Outgoing email settings. To use Gmail or Google Apps, put your Google Apps domain or gmail.com +# as the SMTP_DOMAIN and your Gmail username and password as the SMTP_USER_NAME and SMTP_PASSWORD. +# +# PLEASE NOTE: In order to enable sending real emails via SMTP locally (e.g., when not in the production Rails environment), +# you must also set SEND_EMAIL_IN_DEVELOPMENT to true below. +# +# If you have trouble with port 587 on Gmail, you can also try setting +# SMTP_AUTHENTICATION to login and the SMTP_PORT to 465. + +# FIXME - At least the next four lines +SMTP_DOMAIN=example.com +SMTP_USER_NAME=huginn@mail.example.com +SMTP_PASSWORD=mailpassword +SMTP_SERVER=smtp.example.com +SMTP_PORT=587 +SMTP_AUTHENTICATION=plain +SMTP_ENABLE_STARTTLS_AUTO=true + +# Set to true to send real emails via SMTP when running in the development Rails environment. +# Set to false to have emails intercepted in development and displayed at http://localhost:3000/letter_opener +SEND_EMAIL_IN_DEVELOPMENT=false + +# The address from which system emails will appear to be sent. +# FIXME - Next line +EMAIL_FROM_ADDRESS=huginn@example.com + +########################### +# Agent Logging # +########################### + +# Number of lines of log messages to keep per Agent +AGENT_LOG_LENGTH=200 + +######################################################################################################## +# OAuth Configuration # +# More information at the wiki: https://github.com/cantino/huginn/wiki/Configuring-OAuth-applications # +######################################################################################################## + +TWITTER_OAUTH_KEY= +TWITTER_OAUTH_SECRET= + +THIRTY_SEVEN_SIGNALS_OAUTH_KEY= +THIRTY_SEVEN_SIGNALS_OAUTH_SECRET= + +GITHUB_OAUTH_KEY= +GITHUB_OAUTH_SECRET= + +TUMBLR_OAUTH_KEY= +TUMBLR_OAUTH_SECRET= + +DROPBOX_OAUTH_KEY= +DROPBOX_OAUTH_SECRET= + +WUNDERLIST_OAUTH_KEY= +WUNDERLIST_OAUTH_SECRET= + +EVERNOTE_OAUTH_KEY= +EVERNOTE_OAUTH_SECRET= +# Set to true in development, false in production +USE_EVERNOTE_SANDBOX=true + +############################# +# AWS and Mechanical Turk # +############################# + +# AWS Credentials for MTurk +AWS_ACCESS_KEY_ID="your aws access key id" +AWS_ACCESS_KEY="your aws access key" + +# Set AWS_SANDBOX to true if you're developing Huginn code. +AWS_SANDBOX=false + +######################## +# Various Settings # +######################## + +# Specify the HTTP backend library for Faraday, commonly used by +# WebsiteAgent, RssAgent and PostAgent. You can change this depending +# on the performance and stability you need for your service. Any +# choice other than "typhoeus", "net_http", or "em_http" should +# require you to bundle a corresponding gem via Gemfile. +FARADAY_HTTP_BACKEND=typhoeus + +# Specify the default User-Agent header value for HTTP requests made +# by Agents that allow overriding the User-Agent header value. +DEFAULT_HTTP_USER_AGENT="Huginn - https://github.com/cantino/huginn" + +# Allow JSONPath eval expresions. i.e., $..price[?(@ < 20)] +# You should not allow this on a shared Huginn box because it is not secure. +ALLOW_JSONPATH_EVAL=false + +# Enable this setting to allow insecure Agents like the ShellCommandAgent. Only do this +# when you trust everyone using your Huginn installation. +ENABLE_INSECURE_AGENTS=false + +# Enable this setting to allow second precision schedule in +# SchedulerAgent. By default, the use of the "second" field is +# restricted so that any value other than a single zero (which means +# "on the minute") is disallowed to prevent abuse of service. +ENABLE_SECOND_PRECISION_SCHEDULE=false + +# Specify the scheduler frequency in seconds (default: 0.3). +# Increasing this value will help reduce the use of system resources +# at the expense of time accuracy. +SCHEDULER_FREQUENCY=0.3 + +# Specify the frequency with which the scheduler checks for and cleans up expired events. +# You can use `m` for minutes, `h` for hours, and `d` for days. +EVENT_EXPIRATION_CHECK=6h + +# Use Graphviz for generating diagrams instead of using Google Chart +# Tools. Specify a dot(1) command path built with SVG support +# enabled. +#USE_GRAPHVIZ_DOT=dot + +# Default layout for agent flow diagrams generated by Graphviz. +# Choose from `circo`, `dot` (default), `fdp`, `neato`, `osage`, +# `patchwork`, `sfdp`, or `twopi`. Note that not all layouts are +# supported by Graphviz depending on the build options. +#DIAGRAM_DEFAULT_LAYOUT=dot + +# Timezone. Use `rake time:zones:local` or `rake time:zones:all` to get your zone name +# FIXME - Next line unless you are in the UK +TIMEZONE="London" + +# Number of failed jobs to keep in the database +FAILED_JOBS_TO_KEEP=100 + +# Maximum runtime of background jobs in minutes +DELAYED_JOB_MAX_RUNTIME=2 + +# Amount of seconds for delayed_job to sleep before checking for new jobs +DELAYED_JOB_SLEEP_DELAY=10 + +############################################################### +# Capistrano deployment, read the documentation: # +# https://github.com/cantino/huginn/doc/manual/capistrano.md # +############################################################### + +#CAPISTRANO_DEPLOY_SERVER= +#CAPISTRANO_DEPLOY_USER= +#CAPISTRANO_DEPLOY_REPO_URL= diff --git a/huginn b/huginn new file mode 100644 index 0000000..ec8ef8c --- /dev/null +++ b/huginn @@ -0,0 +1,70 @@ +## Huginn +## +## Lines starting with two hashes (##) are comments with information. +## Lines starting with one hash (#) are configuration parameters that can be uncommented. +## +################################### +## configuration ## +################################### +## +## See installation.md#using-https for additional HTTPS configuration details. + +upstream huginn { + server unix:/home/huginn/huginn/tmp/sockets/unicorn.socket fail_timeout=0; +} + +## Normal HTTP host +server { + listen 0.0.0.0:3000 default_server; + listen [::]:80 ipv6only=on default_server; + server_name localhost; ## Replace this with something like huginn.example.com + server_tokens off; ## Don't show the nginx version number, a security best practice + root /home/huginn/huginn/public; + + ## Increase this if you want to upload large attachments + client_max_body_size 20m; + + ## Individual nginx logs for this Huginn vhost + access_log /var/log/nginx/huginn_access.log; + error_log /var/log/nginx/huginn_error.log; + + location / { + ## Serve static files from defined root folder. + ## @huginn is a named location for the upstream fallback, see below. + try_files $uri $uri/index.html $uri.html @huginn; + } + + ## If a file, which is not found in the root folder is requested, + ## then the proxy passes the request to the upsteam (huginn unicorn). + location @huginn { + ## If you use HTTPS make sure you disable gzip compression + ## to be safe against BREACH attack. + # gzip off; + + proxy_read_timeout 300; + proxy_connect_timeout 300; + proxy_redirect off; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Frame-Options SAMEORIGIN; + + proxy_pass http://huginn; + } + + ## Enable gzip compression as per rails guide: + ## http://guides.rubyonrails.org/asset_pipeline.html#gzip-compression + ## WARNING: If you are using relative urls remove the block below + ## See config/application.rb under "Relative url support" for the list of + ## other files that need to be changed for relative url support + location ~ ^/(assets)/ { + root /home/huginn/huginn/public; + gzip_static on; # to serve pre-gzipped version + expires max; + add_header Cache-Control public; + } + + error_page 502 /502.html; +} diff --git a/provision.sh b/provision.sh new file mode 100755 index 0000000..64b96e7 --- /dev/null +++ b/provision.sh @@ -0,0 +1,112 @@ +#!/bin/bash + +apt-get update +apt-get install -y vim mc screen # not needed, I just prefer to have them around. +update-alternatives --set editor /usr/bin/vim.basic + +# Install the required packages (needed to compile Ruby and native extensions to Ruby gems): +sudo apt-get install -y runit build-essential git zlib1g-dev libyaml-dev libssl-dev libgdbm-dev \ + libreadline-dev libncurses5-dev libffi-dev curl openssh-server checkinstall libxml2-dev libxslt-dev \ + libcurl4-openssl-dev libicu-dev logrotate python-docutils pkg-config cmake nodejs graphviz + +# Remove the old Ruby versions if present: +apt-get remove -y ruby1.8 ruby1.9 + +# Download Ruby and compile it: +mkdir /usr/src/ruby && cd /usr/src/ruby +curl -L http://cache.ruby-lang.org/pub/ruby/2.2/ruby-2.2.3.tar.bz2 | tar xj +cd ruby-2.2.3 +./configure --disable-install-rdoc +make -j`nproc` +make install + +# Install the bundler and foreman gems: +gem install bundler foreman --no-ri --no-rdoc + +# Create a user for Huginn: +adduser --disabled-login --gecos 'Huginn' huginn + +# Install the database packages: +# FIXME (next two lines) +debconf-set-selections <<< 'mysql-server mysql-server/root_password password rootmysqlpassword' +debconf-set-selections <<< 'mysql-server mysql-server/root_password_again password rootmysqlpassword' +apt-get install -y mysql-server-5.5 mysql-server mysql-client libmysqlclient-dev + +# Set up MySQL user +# FIXME: Next line, use the same as DATABASE_USERNAME and DATABASE_PASSWORD from the env file +mysql -u root -prootmysqlpassword <<< "CREATE USER 'huginn'@'localhost' IDENTIFIED BY 'mysqlpassword';" +# FIXME: Next two lines, use the same as 'rootmysqlpassword' above +mysql -u root -prootmysqlpassword <<< "SET storage_engine=INNODB;" +mysql -u root -prootmysqlpassword <<< "GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, INDEX, ALTER, LOCK TABLES ON huginn_production.* TO 'huginn'@'localhost';" + +# Clone the Source +cd /home/huginn +sudo -u huginn -H git clone https://github.com/cantino/huginn.git -b master huginn +cd /home/huginn/huginn +sudo -u huginn mkdir -p log tmp/pids tmp/sockets +chown -R huginn log/ tmp/ +chmod -R u+rwX,go-w log/ tmp/ +chmod -R u+rwX,go-w log/ +chmod -R u+rwX tmp/ +sudo -u huginn -H cp config/unicorn.rb.example config/unicorn.rb + +# Copy .env file into place +sudo -u huginn cp /vagrant/env /home/huginn/huginn/.env +sudo -u huginn -H chmod o-rwx .env + +# Install the Gems +sudo -u huginn -H bundle install --deployment --without development test + +# Create the database +sudo -u huginn -H bundle exec rake db:create RAILS_ENV=production + +# Migrate to the latest version +sudo -u huginn -H bundle exec rake db:migrate RAILS_ENV=production + +# Create admin user and example agents +sudo -u huginn -H bundle exec rake db:seed RAILS_ENV=production + +# Compile Assets +sudo -u huginn -H bundle exec rake assets:precompile RAILS_ENV=production + +# Copy Procfile into place +sudo -u huginn cp /vagrant/Procfile /home/huginn/huginn/Procfile + +# Export the init scripts: +rake production:export + +# Setup Logrotate +cp deployment/logrotate/huginn /etc/logrotate.d/huginn + +# Check it's running +rake production:status + +# Install nginx +apt-get install -y nginx +cp deployment/nginx/huginn /etc/nginx/sites-available/huginn +ln -s /etc/nginx/sites-available/huginn /etc/nginx/sites-enabled/huginn +cp /vagrant/huginn /etc/nginx/sites-available/huginn +rm /etc/nginx/sites-enabled/default +nginx -t +service nginx restart + + + + + + + + + + + + + + + + + + + + +