aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAlec Warner <antarus@gentoo.org>2018-02-18 15:25:04 -0500
committerAlec Warner <antarus@gentoo.org>2018-02-18 15:25:45 -0500
commitc529287ab75f16d3b7cf2445c0122b87e9b161d1 (patch)
treee2bd8af66f7461ebdb3ea20eca84e397bfac015a
parentREDIS_URL should use redis scheme. (diff)
parentpackages_search: raise result limit for USE-flag search (diff)
downloadpackages-5-c529287ab75f16d3b7cf2445c0122b87e9b161d1.tar.gz
packages-5-c529287ab75f16d3b7cf2445c0122b87e9b161d1.tar.bz2
packages-5-c529287ab75f16d3b7cf2445c0122b87e9b161d1.zip
Merge branch 'es-6-compat' into master.v6.0.0
Signed-off-by: Robin Johnson <robbat2@gentoo.org> Collectively this contains a series of commits to move from Elasticsearch v2.X to Elasticsearch v6.0.1. It also contains a bundler update for the ruby environment.
-rw-r--r--Dockerfile30
-rw-r--r--Gemfile.lock184
-rw-r--r--app/models/category.rb8
-rw-r--r--app/models/change.rb12
-rw-r--r--app/models/package.rb28
-rw-r--r--app/models/useflag.rb12
-rw-r--r--app/models/version.rb28
-rw-r--r--config/initializers/elasticsearch.rb9
-rw-r--r--config/initializers/kkuleomi_config.rb.dist5
-rw-r--r--docker-compose.yml61
-rw-r--r--lib/kkuleomi/store.rb47
-rw-r--r--lib/kkuleomi/store/model.rb14
-rw-r--r--lib/kkuleomi/store/models/package_import.rb2
-rw-r--r--lib/kkuleomi/store/models/package_search.rb10
-rw-r--r--lib/kkuleomi/store/models/version_import.rb4
15 files changed, 275 insertions, 179 deletions
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000..d54d9fa
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,30 @@
+FROM gentoo/portage:latest as portage
+FROM gentoo/stage3-amd64
+
+# Need a portage tree to build, use last nights.
+COPY --from=portage /usr/portage /usr/portage
+# Sandbox doesn't work well in docker.
+
+ENV FEATURES="-userpriv -usersandbox -sandbox"
+ENV USE="-bindist"
+
+RUN emerge -C openssh
+RUN emerge net-libs/nodejs
+# Bundler is how we install the ruby stuff.
+RUN emerge dev-ruby/bundler
+
+# Needed for changelogs.
+RUN git clone https://anongit.gentoo.org/git/repo/gentoo.git /mnt/packages-tree/gentoo/
+
+# Copy code into place.
+COPY ./ /var/www/packages.gentoo.org/htdocs/
+WORKDIR /var/www/packages.gentoo.org/htdocs/
+RUN bundler install
+
+# Git clones here.
+RUN cp /var/www/packages.gentoo.org/htdocs/config/secrets.yml.dist /var/www/packages.gentoo.org/htdocs/config/secrets.yml
+RUN sed -i 's/set_me/ENV["SECRET_KEY_BASE"]/'g /var/www/packages.gentoo.org/htdocs/config/secrets.yml
+
+# Precompile our assets.
+RUN rake assets:precompile
+CMD ["bundler", "exec", "thin", "start"]
diff --git a/Gemfile.lock b/Gemfile.lock
index 9b707f5..001d540 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -36,88 +36,85 @@ GEM
minitest (~> 5.1)
thread_safe (~> 0.3, >= 0.3.4)
tzinfo (~> 1.1)
- arel (6.0.3)
+ arel (6.0.4)
axiom-types (0.1.1)
descendants_tracker (~> 0.0.4)
ice_nine (~> 0.11.0)
thread_safe (~> 0.3, >= 0.3.1)
- binding_of_caller (0.7.2)
+ binding_of_caller (0.8.0)
debug_inspector (>= 0.0.1)
- builder (3.2.2)
- byebug (8.2.1)
+ builder (3.2.3)
+ byebug (10.0.0)
coercible (1.0.0)
descendants_tracker (~> 0.0.1)
- coffee-rails (4.1.1)
- coffee-script (>= 2.2.0)
- railties (>= 4.0.0, < 5.1.x)
- coffee-script (2.4.1)
- coffee-script-source
- execjs
- coffee-script-source (1.10.0)
- concurrent-ruby (1.0.0)
- connection_pool (2.2.0)
- daemons (1.2.3)
- debug_inspector (0.0.2)
+ concurrent-ruby (1.0.5)
+ connection_pool (2.2.1)
+ crass (1.0.3)
+ daemons (1.2.6)
+ debug_inspector (0.0.3)
descendants_tracker (0.0.4)
thread_safe (~> 0.3, >= 0.3.1)
- elasticsearch (1.0.15)
- elasticsearch-api (= 1.0.15)
- elasticsearch-transport (= 1.0.15)
- elasticsearch-api (1.0.15)
+ elasticsearch (5.0.4)
+ elasticsearch-api (= 5.0.4)
+ elasticsearch-transport (= 5.0.4)
+ elasticsearch-api (5.0.4)
multi_json
- elasticsearch-model (0.1.8)
+ elasticsearch-model (5.0.2)
activesupport (> 3)
- elasticsearch (> 0.4)
+ elasticsearch (~> 5)
hashie
- elasticsearch-persistence (0.1.8)
- activemodel (> 3)
- activesupport (> 3)
- elasticsearch (> 0.4)
- elasticsearch-model (>= 0.1)
+ elasticsearch-persistence (5.0.2)
+ activemodel (> 4)
+ activesupport (> 4)
+ elasticsearch (~> 5)
+ elasticsearch-model (~> 5)
hashie
virtus
- elasticsearch-rails (0.1.8)
- elasticsearch-transport (1.0.15)
+ elasticsearch-rails (5.0.2)
+ elasticsearch-transport (5.0.4)
faraday
multi_json
equalizer (0.0.11)
erubis (2.7.0)
- eventmachine (1.0.9.1)
- execjs (2.6.0)
- faraday (0.9.2)
+ eventmachine (1.2.5)
+ execjs (2.7.0)
+ faraday (0.14.0)
multipart-post (>= 1.2, < 3)
- git (1.2.9.1)
- globalid (0.3.6)
- activesupport (>= 4.1.0)
- hashie (3.4.3)
- i18n (0.7.0)
- ice_nine (0.11.1)
- jbuilder (2.4.0)
- activesupport (>= 3.0.0, < 5.1)
- multi_json (~> 1.2)
- jquery-rails (4.1.0)
- rails-dom-testing (~> 1.0)
+ ffi (1.9.21)
+ git (1.3.0)
+ globalid (0.4.1)
+ activesupport (>= 4.2.0)
+ hashie (3.5.7)
+ i18n (0.9.5)
+ concurrent-ruby (~> 1.0)
+ ice_nine (0.11.2)
+ jbuilder (2.7.0)
+ activesupport (>= 4.2.0)
+ multi_json (>= 1.2)
+ jquery-rails (4.3.1)
+ rails-dom-testing (>= 1, < 3)
railties (>= 4.2.0)
thor (>= 0.14, < 2.0)
- json (1.8.3)
- loofah (2.0.3)
+ json (1.8.6)
+ loofah (2.2.0)
+ crass (~> 1.0.2)
nokogiri (>= 1.5.9)
- mail (2.6.3)
- mime-types (>= 1.16, < 3)
- mime-types (2.99)
- mini_portile2 (2.0.0)
- minitest (5.8.4)
- multi_json (1.11.2)
+ mail (2.7.0)
+ mini_mime (>= 0.1.1)
+ mini_mime (1.0.0)
+ mini_portile2 (2.3.0)
+ minitest (5.11.3)
+ multi_json (1.13.1)
multipart-post (2.0.0)
- nokogiri (1.6.7.2)
- mini_portile2 (~> 2.0.0.rc2)
- octicons (2.1.0)
+ nokogiri (1.8.2)
+ mini_portile2 (~> 2.3.0)
+ octicons (5.2.0)
nokogiri (>= 1.6.3.1)
- octicons_helper (2.1.0)
- octicons (~> 2.0)
+ octicons_helper (4.2.0)
+ octicons (= 5.2.0)
rails
- parallel (1.6.1)
- rack (1.6.4)
+ parallel (1.12.1)
+ rack (1.6.8)
rack-protection (1.5.3)
rack
rack-test (0.6.3)
@@ -135,9 +132,9 @@ GEM
sprockets-rails
rails-deprecated_sanitizer (1.0.3)
activesupport (>= 4.2.0.alpha)
- rails-dom-testing (1.0.7)
- activesupport (>= 4.2.0.beta, < 5.0)
- nokogiri (~> 1.6.0)
+ rails-dom-testing (1.0.9)
+ activesupport (>= 4.2.0, < 5.0)
+ nokogiri (~> 1.6)
rails-deprecated_sanitizer (>= 1.0.1)
rails-html-sanitizer (1.0.3)
loofah (~> 2.0)
@@ -146,58 +143,66 @@ GEM
activesupport (= 4.2.5.1)
rake (>= 0.8.7)
thor (>= 0.18.1, < 2.0)
- rake (10.5.0)
- rdiscount (2.1.8)
- rdoc (4.2.1)
- json (~> 1.4)
- redis (3.2.2)
- ruby-progressbar (1.7.5)
- sass (3.4.21)
- sass-rails (5.0.4)
- railties (>= 4.0.0, < 5.0)
+ rake (12.3.0)
+ rb-fsevent (0.10.2)
+ rb-inotify (0.9.10)
+ ffi (>= 0.5.0, < 2)
+ rdiscount (2.2.0.1)
+ rdoc (4.3.0)
+ redis (4.0.1)
+ ruby-progressbar (1.9.0)
+ sass (3.5.5)
+ sass-listen (~> 4.0.0)
+ sass-listen (4.0.0)
+ rb-fsevent (~> 0.9, >= 0.9.4)
+ rb-inotify (~> 0.9, >= 0.9.7)
+ sass-rails (5.0.7)
+ railties (>= 4.0.0, < 6)
sass (~> 3.1)
sprockets (>= 2.8, < 4.0)
sprockets-rails (>= 2.0, < 4.0)
tilt (>= 1.1, < 3)
- sdoc (0.4.1)
+ sdoc (0.4.2)
json (~> 1.7, >= 1.7.7)
rdoc (~> 4.0)
- sidekiq (4.0.2)
+ sidekiq (5.1.1)
concurrent-ruby (~> 1.0)
connection_pool (~> 2.2, >= 2.2.0)
- redis (~> 3.2, >= 3.2.1)
- sinatra (1.4.7)
+ rack-protection (>= 1.5.0)
+ redis (>= 3.3.5, < 5)
+ sinatra (1.4.8)
rack (~> 1.5)
rack-protection (~> 1.4)
tilt (>= 1.3, < 3)
- spring (1.6.2)
- sprockets (3.5.2)
+ spring (2.0.2)
+ activesupport (>= 4.2)
+ sprockets (3.7.1)
concurrent-ruby (~> 1.0)
rack (> 1, < 3)
- sprockets-rails (3.0.0)
+ sprockets-rails (3.2.1)
actionpack (>= 4.0)
activesupport (>= 4.0)
sprockets (>= 3.0.0)
- thin (1.6.4)
+ thin (1.7.2)
daemons (~> 1.0, >= 1.0.9)
eventmachine (~> 1.0, >= 1.0.4)
- rack (~> 1.0)
- thor (0.19.1)
- thread_safe (0.3.5)
- tilt (2.0.2)
- turbolinks (2.5.3)
- coffee-rails
- tzinfo (1.2.2)
+ rack (>= 1, < 3)
+ thor (0.20.0)
+ thread_safe (0.3.6)
+ tilt (2.0.8)
+ turbolinks (5.1.0)
+ turbolinks-source (~> 5.1)
+ turbolinks-source (5.1.0)
+ tzinfo (1.2.5)
thread_safe (~> 0.1)
- uglifier (2.7.2)
- execjs (>= 0.3.0)
- json (>= 1.8.0)
+ uglifier (4.1.6)
+ execjs (>= 0.3.0, < 3)
virtus (1.0.5)
axiom-types (~> 0.1)
coercible (~> 1.0)
descendants_tracker (~> 0.0, >= 0.0.3)
equalizer (~> 0.0, >= 0.0.9)
- web-console (2.2.1)
+ web-console (2.3.0)
activemodel (>= 4.0)
binding_of_caller (>= 0.7.2)
railties (>= 4.0)
@@ -228,3 +233,6 @@ DEPENDENCIES
turbolinks
uglifier (>= 1.3.0)
web-console (~> 2.0)
+
+BUNDLED WITH
+ 1.11.2
diff --git a/app/models/category.rb b/app/models/category.rb
index 5fa31ea..f629bde 100644
--- a/app/models/category.rb
+++ b/app/models/category.rb
@@ -2,11 +2,11 @@ class Category
include Elasticsearch::Persistence::Model
include Kkuleomi::Store::Model
- index_name "packages-#{Rails.env}"
+ index_name "categories-#{Rails.env}"
- attribute :name, String, mapping: { index: 'not_analyzed' }
- attribute :description, String
- attribute :metadata_hash, String, mapping: { index: 'not_analyzed' }
+ attribute :name, String, mapping: { type: 'keyword' }
+ attribute :description, String, mapping: { type: 'text' }
+ attribute :metadata_hash, String, mapping: { type: 'text' }
# Determines if the document model needs an update from the repository model
#
diff --git a/app/models/change.rb b/app/models/change.rb
index 9ffe258..6eaf00c 100644
--- a/app/models/change.rb
+++ b/app/models/change.rb
@@ -2,12 +2,12 @@ class Change
include Elasticsearch::Persistence::Model
include Kkuleomi::Store::Model
- index_name "packages-#{Rails.env}"
+ index_name "change-#{Rails.env}"
- attribute :package, String, mapping: { index: 'not_analyzed' }
- attribute :category, String, mapping: { index: 'not_analyzed' }
- attribute :change_type, String, mapping: { index: 'not_analyzed' }
- attribute :version, String, mapping: { index: 'not_analyzed' }
- attribute :arches, String, mapping: { index: 'not_analyzed' }
+ attribute :package, String, mapping: { type: 'keyword' }
+ attribute :category, String, mapping: { type: 'keyword' }
+ attribute :change_type, String, mapping: { type: 'keyword' }
+ attribute :version, String, mapping: { type: 'keyword' }
+ attribute :arches, String, mapping: { type: 'keyword' }
attribute :commit, Hash, default: {}, mapping: { type: 'object' }
end
diff --git a/app/models/package.rb b/app/models/package.rb
index 24cbd8b..7ad3cbe 100644
--- a/app/models/package.rb
+++ b/app/models/package.rb
@@ -6,19 +6,23 @@ class Package
index_name "packages-#{Rails.env}"
- attribute :category, String, mapping: { index: 'not_analyzed' }
- attribute :name, String, mapping: { index: 'not_analyzed' }
- attribute :name_sort, String, mapping: { index: 'not_analyzed' }
- attribute :atom, String, mapping: { index: 'not_analyzed' }
- attribute :description, String
- attribute :longdescription, String
- attribute :homepage, String, default: [], mapping: { index: 'not_analyzed' }
- attribute :license, String, mapping: { index: 'not_analyzed' }
- attribute :licenses, String, default: [], mapping: { index: 'not_analyzed' }
- attribute :herds, String, default: [], mapping: { index: 'not_analyzed' }
+ raw_fields = {
+ type: 'keyword'
+ }
+
+ attribute :category, String, mapping: raw_fields
+ attribute :name, String, mapping: raw_fields
+ attribute :name_sort, String, mapping: raw_fields
+ attribute :atom, String, mapping: raw_fields
+ attribute :description, String, mapping: { type: 'text' }
+ attribute :longdescription, String, mapping: { type: 'text' }
+ attribute :homepage, String, default: [], mapping: raw_fields
+ attribute :license, String, mapping: raw_fields
+ attribute :licenses, String, default: [], mapping: raw_fields
+ attribute :herds, String, default: [], mapping: raw_fields
attribute :maintainers, Array, default: [], mapping: { type: 'object' }
attribute :useflags, Hash, default: {}, mapping: { type: 'object' }
- attribute :metadata_hash, String, mapping: { index: 'not_analyzed' }
+ attribute :metadata_hash, String, mapping: raw_fields
def category_model
@category_model ||= Category.find_by(:name, category)
@@ -40,7 +44,7 @@ class Package
end
def versions
- @versions ||= Version.find_all_by_parent(self, sort: { sort_key: { order: 'asc' } })
+ @versions ||= Version.find_all_by(:package, atom, sort: { sort_key: { order: 'asc' } })
end
def latest_version
diff --git a/app/models/useflag.rb b/app/models/useflag.rb
index 1139c97..131a89c 100644
--- a/app/models/useflag.rb
+++ b/app/models/useflag.rb
@@ -2,13 +2,13 @@ class Useflag
include Elasticsearch::Persistence::Model
include Kkuleomi::Store::Model
- index_name "packages-#{Rails.env}"
+ index_name "useflags-#{Rails.env}"
- attribute :name, String, mapping: { index: 'not_analyzed' }
- attribute :description, String
- attribute :atom, String, mapping: { index: 'not_analyzed' }
- attribute :scope, String, mapping: { index: 'not_analyzed' }
- attribute :use_expand_prefix, String, mapping: { index: 'not_analyzed' }
+ attribute :name, String, mapping: { type: 'keyword' }
+ attribute :description, String, mapping: { type: 'text' }
+ attribute :atom, String, mapping: { type: 'keyword' }
+ attribute :scope, String, mapping: { type: 'keyword' }
+ attribute :use_expand_prefix, String, mapping: { type: 'keyword' }
def all_fields
[:name, :description, :atom, :scope, :use_expand_prefix]
diff --git a/app/models/version.rb b/app/models/version.rb
index 1dc28f8..a457c2c 100644
--- a/app/models/version.rb
+++ b/app/models/version.rb
@@ -3,21 +3,21 @@ class Version
include Kkuleomi::Store::Model
include Kkuleomi::Store::Models::VersionImport
- index_name "packages-#{Rails.env}"
-
- attribute :version, String, mapping: { index: 'not_analyzed' }
- attribute :package, String, mapping: { index: 'not_analyzed' }
- attribute :atom, String, mapping: { index: 'not_analyzed' }
- attribute :sort_key, Integer, mapping: { index: 'not_analyzed' }
- attribute :slot, String, mapping: { index: 'not_analyzed' }
- attribute :subslot, String, mapping: { index: 'not_analyzed' }
- attribute :eapi, String, mapping: { index: 'not_analyzed' }
- attribute :keywords, String, mapping: { index: 'not_analyzed' }
+ index_name "versions-#{Rails.env}"
+
+ attribute :version, String, mapping: { type: 'keyword' }
+ attribute :package, String, mapping: { type: 'keyword' }
+ attribute :atom, String, mapping: { type: 'keyword' }
+ attribute :sort_key, Integer, mapping: { type: 'keyword' }
+ attribute :slot, String, mapping: { type: 'keyword' }
+ attribute :subslot, String, mapping: { type: 'keyword' }
+ attribute :eapi, String, mapping: { type: 'keyword' }
+ attribute :keywords, String, mapping: { type: 'keyword' }
attribute :masks, Array, default: [], mapping: { type: 'object' }
- attribute :use, String, default: [], mapping: { index: 'not_analyzed' }
- attribute :restrict, String, default: [], mapping: { index: 'not_analyzed' }
- attribute :properties, String, default: [], mapping: { index: 'not_analyzed' }
- attribute :metadata_hash, String, mapping: { index: 'not_analyzed' }
+ attribute :use, String, default: [], mapping: { type: 'keyword' }
+ attribute :restrict, String, default: [], mapping: { type: 'keyword' }
+ attribute :properties, String, default: [], mapping: { type: 'keyword' }
+ attribute :metadata_hash, String, mapping: { type: 'keyword' }
# Returns the keywording state on a given architecture
#
diff --git a/config/initializers/elasticsearch.rb b/config/initializers/elasticsearch.rb
index 9be3660..e873162 100644
--- a/config/initializers/elasticsearch.rb
+++ b/config/initializers/elasticsearch.rb
@@ -1,10 +1,9 @@
require 'elasticsearch/persistence/model'
-Elasticsearch::Persistence.client = Elasticsearch::Client.new host: ENV['ELASTICSEARCH_URL'] || 'localhost:9200'
-
-if Rails.env.development?
+Elasticsearch::Persistence.client = Elasticsearch::Client.new host: ENV['ELASTICSEARCH_URL'] || 'elasticsearch:9300'
+if Rails.env.development? or ENV['RAILS_DEBUG']
logger = ActiveSupport::Logger.new(STDERR)
- logger.level = Logger::INFO
+ logger.level = Logger::DEBUG
logger.formatter = proc { |s, d, p, m| "\e[2m#{m}\n\e[0m" }
Elasticsearch::Persistence.client.transport.logger = logger
-end \ No newline at end of file
+end
diff --git a/config/initializers/kkuleomi_config.rb.dist b/config/initializers/kkuleomi_config.rb.dist
index 0bbf86e..f6d4937 100644
--- a/config/initializers/kkuleomi_config.rb.dist
+++ b/config/initializers/kkuleomi_config.rb.dist
@@ -11,8 +11,11 @@ KKULEOMI_FIRST_COMMIT='c1de71edb35b118c3244c0d9d1b3f97c93d41969'
# Where does git(1) live?
KKULEOMI_GIT='git'
+# GIT enabled?
+KKULEOMI_DISABLE_GIT=false
+
# Who gets feedback
-KKULEOMI_FEEDBACK_RECIPIENT='a3li@gentoo.org'
+KKULEOMI_FEEDBACK_RECIPIENT='gpackages@gentoo.org'
# Default arches.
KKULEOMI_ARCHES=%w(amd64 x86 alpha arm hppa ia64 ppc ppc64 sparc)
diff --git a/docker-compose.yml b/docker-compose.yml
new file mode 100644
index 0000000..7c10720
--- /dev/null
+++ b/docker-compose.yml
@@ -0,0 +1,61 @@
+
+version: '2'
+# p.g.o consists nominally of:
+# A service to sync the tree to $TREE_DIR && update the index.
+# The index updater simply submits work items into sidekiq.
+# ElasticSearch to managed updates and serve documents.
+# SideKiq requires redis.
+#
+# To this end we define 4 containers.
+# - HTTP Serving: Serves the site, and the /sidekiq endpoing.
+# - Elasticsearch: runs the elasticsearch service.
+# - Redis: Run Redis for Sidekiq.
+# - Refresher: Syncs the package tree and pushes updates into sidekiq.
+services:
+ http-serving:
+ # Build from Dockerfile in .
+ build: .
+ ports:
+ - 5000
+ environment:
+ # "Redis:port" and "elasticsearch:port" refer to sibling containers.
+ - REDIS_PROVIDER=REDIS_URL
+ - REDIS_URL=redis://redis:6379
+ - ELASTICSEARCH_URL=elasticsearch:9200
+ - RAILS_SERVE_STATIC_FILES=1
+ - RAILS_ENV=production
+ - MEMCACHE_URL="memcache:11211"
+ - SECRET_KEY_BASE=6c9710aeb74dd88ff1d1b8f4bd6d7d8e0f340905d0974400fffd7246714aa703cf7bf4a98c0bc90317a3b803b82c0f9371e18ada19fc4eed9d6118077a249f50
+ depends_on:
+ - redis
+ - elasticsearch
+ command: bundle exec thin start -p 5000
+ sidekiq:
+ build: .
+ environment:
+ - RAILS_ENV=production
+ - RAILS_SERVE_STATIC_FILES=1
+ - REDIS_URL=redis://redis:6379
+ - MEMCACHE_URL="memcache:11211"
+ - ELASTICSEARCH_URL=elasticsearch:9200
+ - SECRET_KEY_BASE=6c9710aeb74dd88ff1d1b8f4bd6d7d8e0f340905d0974400fffd7246714aa703cf7bf4a98c0bc90317a3b803b82c0f9371e18ada19fc4eed9d6118077a249f50
+ depends_on:
+ - redis
+ - elasticsearch
+ command: bundle exec sidekiq -c 5
+ memcache:
+ image: memcached:latest
+ ports:
+ - 11211
+ elasticsearch:
+ # TODO(antarus): We should build a docker image for this based on gentoo.
+ image: docker.elastic.co/elasticsearch/elasticsearch:6.0.1
+ # Run in single-node config.
+ environment:
+ - discovery.type=single-node
+ ports:
+ - 9200
+ redis:
+ image: redis:4.0.6
+ ports:
+ - 6379
diff --git a/lib/kkuleomi/store.rb b/lib/kkuleomi/store.rb
index 4baf2c4..150a64f 100644
--- a/lib/kkuleomi/store.rb
+++ b/lib/kkuleomi/store.rb
@@ -4,26 +4,15 @@ module Kkuleomi::Store
end
def self.create_index(force = false)
- client = Category.gateway.client
- index_name = Category.index_name
+ types = [
+ Category,
+ Package,
+ Version,
+ Change,
+ Useflag,
+ ]
- settings_list = [
- Category.settings.to_hash,
- Package.settings.to_hash,
- Version.settings.to_hash,
- Change.settings.to_hash,
- Useflag.settings.to_hash
- ]
-
- mappings_list = [
- Category.mappings.to_hash,
- Package.mappings.to_hash,
- Version.mappings.to_hash,
- Change.mappings.to_hash,
- Useflag.mappings.to_hash
- ]
-
- settings = {
+ base_settings = {
analysis: {
filter: {
autocomplete_filter: {
@@ -39,15 +28,19 @@ module Kkuleomi::Store
filter: %w(lowercase autocomplete_filter)
}
}
- }
+ },
+ mapping: { total_fields: { limit: 25000 } }
}
- settings_list.each { |setting| settings.merge! setting }
-
- mappings = {}
- mappings_list.each { |mapping| mappings.merge! mapping }
-
- client.indices.delete(index: index_name) rescue nil if force
- client.indices.create(index: index_name, body: { settings: settings, mappings: mappings })
+ # In ES 1.5, we could use 1 mega-index. But in ES6, each model needs its own.
+ types.each { |type|
+ client = type.gateway.client
+ client.indices.delete(index: type.index_name) rescue nil if force
+ body = {
+ settings: type.settings.to_hash.merge(base_settings),
+ mappings: type.mappings.to_hash
+ }
+ client.indices.create(index: type.index_name, body: body)
+ }
end
end
diff --git a/lib/kkuleomi/store/model.rb b/lib/kkuleomi/store/model.rb
index 8d406bf..653884b 100644
--- a/lib/kkuleomi/store/model.rb
+++ b/lib/kkuleomi/store/model.rb
@@ -9,9 +9,7 @@ module Kkuleomi::Store::Model
def find_all_by(field, value, opts = {})
search({
size: 10_000,
- query: {
- filtered: { filter: { term: { field => value } } }
- }
+ query: { bool: { filter: { term: { field => value } } } }
}.merge(opts))
end
@@ -22,7 +20,7 @@ module Kkuleomi::Store::Model
search({
query: {
- filtered: { filter: { bool: { must: filter_args } } }
+ bool: { filter: { bool: { must: filter_args } } }
},
size: 10_000
}.merge(opts))
@@ -36,14 +34,14 @@ module Kkuleomi::Store::Model
search(opts.merge(
size: 10_000,
query: {
- filtered: {
+ bool: {
filter: {
has_parent: {
- type: parent.class.document_type,
- filter: { term: { _id: parent.id } }
+ parent_type: parent.class.document_type,
+ query: { term: { _id: parent.id } }
}
},
- query: { match_all: {} }
+ must: { match_all: {} }
}
}
))
diff --git a/lib/kkuleomi/store/models/package_import.rb b/lib/kkuleomi/store/models/package_import.rb
index 3b12d8a..5de425a 100644
--- a/lib/kkuleomi/store/models/package_import.rb
+++ b/lib/kkuleomi/store/models/package_import.rb
@@ -103,7 +103,7 @@ module Kkuleomi::Store::Models::PackageImport
end
def import_versions!(package_model, ebuilds, options)
- index_v = Hash[Version.find_all_by_parent(self).map { |v| [v.version, v] }]
+ index_v = Hash[Version.find_all_by(:package, self.name).map { |v| [v.version, v] }]
model_v = Hash[ebuilds.map { |v| [v.version, v] }]
index_keys = index_v.keys
diff --git a/lib/kkuleomi/store/models/package_search.rb b/lib/kkuleomi/store/models/package_search.rb
index 150b0e9..263eb25 100644
--- a/lib/kkuleomi/store/models/package_search.rb
+++ b/lib/kkuleomi/store/models/package_search.rb
@@ -30,9 +30,10 @@ module Kkuleomi::Store::Models::PackageSearch
# Results are aggregated by package atoms.
def find_atoms_by_useflag(useflag)
Version.search(
+ size: 10000, # default limit is 10.
query: {
- filtered: {
- query: { match_all: {} },
+ bool: {
+ must: { match_all: {} },
filter: { term: { use: useflag } }
}
},
@@ -40,8 +41,7 @@ module Kkuleomi::Store::Models::PackageSearch
group_by_package: {
terms: {
field: 'package',
- size: 0,
- order: { '_term' => 'asc' }
+ order: { '_key' => 'asc' }
}
}
},
@@ -147,7 +147,7 @@ module Kkuleomi::Store::Models::PackageSearch
[
{
filter: { term: { category: 'virtual' } },
- boost_factor: 0.6
+ weight: 0.6
}
]
end
diff --git a/lib/kkuleomi/store/models/version_import.rb b/lib/kkuleomi/store/models/version_import.rb
index b4e4caf..b65b683 100644
--- a/lib/kkuleomi/store/models/version_import.rb
+++ b/lib/kkuleomi/store/models/version_import.rb
@@ -38,7 +38,7 @@ module Kkuleomi::Store::Models::VersionImport
self.masks = Portage::Util::Masks.for(ebuild_model)
self.metadata_hash = ebuild_model.metadata_hash
- save(parent: parent_package.id)
+ save()
# If keywords changed, calculate changes and record as needed (but only do that if we should)
unless options[:suppress_change_objects]
@@ -60,7 +60,7 @@ module Kkuleomi::Store::Models::VersionImport
# @param [Package] parent Parent package model
def set_sort_key!(key, parent)
self.sort_key = key
- save(parent: parent.id)
+ save()
end
def strip_useflag_defaults(flags)