Merge pull request pypi#739 from dstufft/search

Implement a basic search
CarlosDomingues · Nov 8, 2015 · ae6bfce · ae6bfce
2 parents 05e4a16 + 4f9c6e6
commit ae6bfce
Show file tree

Hide file tree

Showing 13 changed files with 283 additions and 4 deletions.
diff --git a/requirements.txt b/requirements.txt
@@ -32,6 +32,7 @@ markupsafe==0.23          # via jinja2, mako, pyramid-jinja2
 msgpack-python==0.4.6
 newrelic==2.56.0.42
 packaging==15.3
+paginate==0.5.2
 passlib==1.6.5
 pastedeploy==1.5.2        # via pyramid
 psycopg2==2.6.1

diff --git a/setup.py b/setup.py
@@ -83,6 +83,7 @@
         "Jinja2>=2.8",
         "msgpack-python",
         "packaging>=15.2",
+        "paginate>=0.5.2",
         "passlib>=1.6.4",
         "psycopg2",
         "pyramid>=1.6a1",

diff --git a/tests/unit/cli/search/test_reindex.py b/tests/unit/cli/search/test_reindex.py
@@ -39,6 +39,7 @@ def test_project_docs(db_session):
             "_type": "project",
             "_source": {
                 "name": p.name,
+                "normalized_name": p.normalized_name,
                 "version": [r.version for r in prs],
             },
         }

diff --git a/tests/unit/test_routes.py b/tests/unit/test_routes.py
@@ -67,6 +67,7 @@ def add_xmlrpc_endpoint(endpoint, pattern, header, read_only=False):
             traverse="/{name}",
             read_only=True,
         ),
+        pretend.call("search", "/search/", read_only=True),
         pretend.call(
             "accounts.profile",
             "/user/{username}/",

diff --git a/tests/unit/test_views.py b/tests/unit/test_views.py
@@ -13,9 +13,12 @@
 import datetime
 
 import pretend
+import pytest
 
+from warehouse import views
 from warehouse.views import (
     forbidden, index, httpexception_view, robotstxt, current_user_indicator,
+    search,
 )
 
 from ..common.db.packaging import (
@@ -87,3 +90,69 @@ def test_index(self, db_request):
 
 def test_esi_current_user_indicator():
     assert current_user_indicator(pretend.stub()) == {}
+
+
+class TestSearch:
+
+    @pytest.mark.parametrize("page", [None, 1, 5])
+    def test_with_a_query(self, monkeypatch, page):
+        params = {"q": "foo bar"}
+        if page is not None:
+            params["page"] = page
+        query = pretend.stub()
+        request = pretend.stub(
+            es=pretend.stub(
+                query=pretend.call_recorder(lambda *a, **kw: query),
+            ),
+            params=params,
+        )
+
+        page_obj = pretend.stub()
+        page_cls = pretend.call_recorder(lambda *a, **kw: page_obj)
+        monkeypatch.setattr(views, "ElasticsearchPage", page_cls)
+
+        url_maker = pretend.stub()
+        url_maker_factory = pretend.call_recorder(lambda request: url_maker)
+        monkeypatch.setattr(views, "paginate_url_factory", url_maker_factory)
+
+        assert search(request) == {"page": page_obj}
+        assert page_cls.calls == [
+            pretend.call(query, url_maker=url_maker, page=page or 1),
+        ]
+        assert url_maker_factory.calls == [pretend.call(request)]
+        assert request.es.query.calls == [
+            pretend.call(
+                "multi_match",
+                query="foo bar",
+                fields=[
+                    "name", "version", "author", "author_email", "maintainer",
+                    "maintainer_email", "home_page", "license", "summary",
+                    "description", "keywords", "platform", "download_url",
+                ],
+            ),
+        ]
+
+    @pytest.mark.parametrize("page", [None, 1, 5])
+    def test_without_a_query(self, monkeypatch, page):
+        params = {}
+        if page is not None:
+            params["page"] = page
+        query = pretend.stub()
+        request = pretend.stub(
+            es=pretend.stub(query=lambda: query),
+            params=params,
+        )
+
+        page_obj = pretend.stub()
+        page_cls = pretend.call_recorder(lambda *a, **kw: page_obj)
+        monkeypatch.setattr(views, "ElasticsearchPage", page_cls)
+
+        url_maker = pretend.stub()
+        url_maker_factory = pretend.call_recorder(lambda request: url_maker)
+        monkeypatch.setattr(views, "paginate_url_factory", url_maker_factory)
+
+        assert search(request) == {"page": page_obj}
+        assert page_cls.calls == [
+            pretend.call(query, url_maker=url_maker, page=page or 1),
+        ]
+        assert url_maker_factory.calls == [pretend.call(request)]
diff --git a/tests/unit/utils/test_paginate.py b/tests/unit/utils/test_paginate.py
@@ -0,0 +1,103 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pretend
+import pytest
+
+from webob.multidict import MultiDict
+
+from warehouse.utils import paginate
+
+
+class FakeResult:
+
+    def __init__(self, data, total):
+        self.data = data
+        self.total = total
+
+    @property
+    def hits(self):
+        return pretend.stub(total=self.total)
+
+    def __iter__(self):
+        for i in self.data:
+            yield i
+
+
+class FakeQuery:
+
+    def __init__(self, fake):
+        self.fake = fake
+        self.range = slice(None)
+
+    def __getitem__(self, range):
+        self.range = range
+        return self
+
+    @property
+    def results(self):
+        return pretend.stub(hits=pretend.stub(total=len(self.fake)))
+
+    def execute(self):
+        return FakeResult(self.fake[self.range], len(self.fake))
+
+
+class TestElasticsearchWrapper:
+
+    def test_slices_and_length(self):
+        wrapper = paginate._ElasticsearchWrapper(FakeQuery([1, 2, 3, 4, 5, 6]))
+        assert wrapper[1:3] == [2, 3]
+        assert len(wrapper) == 6
+
+    def test_second_slice_fails(self):
+        wrapper = paginate._ElasticsearchWrapper(FakeQuery([1, 2, 3, 4, 5, 6]))
+        wrapper[1:3]
+
+        with pytest.raises(RuntimeError):
+            wrapper[1:3]
+
+    def test_len_before_slice_fails(self):
+        wrapper = paginate._ElasticsearchWrapper(FakeQuery([1, 2, 3, 4, 5, 6]))
+
+        with pytest.raises(RuntimeError):
+            len(wrapper)
+
+
+def test_elasticsearch_page_has_wrapper(monkeypatch):
+    page_obj = pretend.stub()
+    page_cls = pretend.call_recorder(lambda *a, **kw: page_obj)
+    monkeypatch.setattr(paginate, "Page", page_cls)
+
+    assert paginate.ElasticsearchPage("first", second="foo") is page_obj
+    assert page_cls.calls == [
+        pretend.call(
+            "first",
+            second="foo",
+            wrapper_class=paginate._ElasticsearchWrapper,
+        ),
+    ]
+
+
+def test_paginate_url(pyramid_request):
+    pyramid_request.GET = MultiDict(pyramid_request.GET)
+    pyramid_request.GET["foo"] = "bar"
+
+    url = pretend.stub()
+    pyramid_request.current_route_path = \
+        pretend.call_recorder(lambda _query: url)
+
+    url_maker = paginate.paginate_url_factory(pyramid_request)
+
+    assert url_maker(5) is url
+    assert pyramid_request.current_route_path.calls == [
+        pretend.call(_query=[("foo", "bar"), ("page", 5)]),
+    ]
diff --git a/warehouse/packaging/search.py b/warehouse/packaging/search.py
@@ -26,6 +26,7 @@
 class Project(DocType):
 
     name = String()
+    normalized_name = String(index="not_analyzed")
     version = String(index="not_analyzed", multi=True)
     summary = String(analyzer="snowball")
     description = String(analyzer="snowball")
@@ -47,6 +48,7 @@ class Meta:
     def from_db(cls, release):
         obj = cls(meta={"id": release.project.normalized_name})
         obj["name"] = release.project.name
+        obj["normalized_name"] = release.project.normalized_name
         obj["version"] = [r.version for r in release.project.releases]
         obj["summary"] = release.summary
         obj["description"] = release.description

diff --git a/warehouse/routes.py b/warehouse/routes.py
@@ -36,6 +36,9 @@ def includeme(config):
         read_only=True,
     )
 
+    # Search Routes
+    config.add_route("search", "/search/", read_only=True)
+
     # Accounts
     config.add_route(
         "accounts.profile",

diff --git a/warehouse/templates/base.html b/warehouse/templates/base.html
@@ -33,8 +33,8 @@
 
         <div class="navigation-tools">
           <div class="search-bar">
-            <form role="search" action="TODO" method="GET">
-              <input type="search" placeholder="{{ _('Enter Search') }}" />
+            <form role="search" action="{{ request.route_path('search') }}" method="GET">
+              <input type="search" name="q" placeholder="{{ _('Enter Search') }}" />
               <button type="submit">
                 <img src="{{ request.static_path('warehouse:static/images/search-icon.png') }}" alt="{{ _('Search') }}">
               </button>

diff --git a/warehouse/templates/index.html b/warehouse/templates/index.html
@@ -8,8 +8,8 @@
             <p>{{ _('For up to date best practices about packaging read <a href="%(packaging_guide_url)s">packaging.python.org</a>.', packaging_guide_url='https://packaging.python.org/en/latest/') }}</p>
         </div>
 
-        <form class="search-bar" role="search" action="TODO" method="GET">
-          <input type="search" placeholder="{{ _('Search packages...') }}" />
+        <form class="search-bar" role="search" action="{{ request.route_path('search') }}" method="GET">
+          <input type="search" name="q" placeholder="{{ _('Search packages...') }}" />
           <button type="submit">
             <img src="{{ request.static_path('warehouse:static/images/search-icon.png') }}" alt="Search Icon">
           </button>

diff --git a/warehouse/templates/search/results.html b/warehouse/templates/search/results.html
@@ -0,0 +1,12 @@
+{% extends "base.html" %}
+
+{% block content %}
+<ol>
+  {% for item in page.items %}
+  <li><a href="{{ request.route_path('packaging.project', name=item.normalized_name) }}">{{ item.name }}</a> {{ item.summary }}</li>
+  {% endfor %}
+</ol>
+
+{{ page.pager()|safe }}
+
+{% endblock %}
diff --git a/warehouse/utils/paginate.py b/warehouse/utils/paginate.py
@@ -0,0 +1,50 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from paginate import Page
+
+
+class _ElasticsearchWrapper:
+
+    def __init__(self, query):
+        self.query = query
+        self.results = None
+
+    def __getitem__(self, range):
+        if self.results is not None:
+            raise RuntimeError("Cannot reslice after having already sliced.")
+        self.results = self.query[range].execute()
+
+        return list(self.results)
+
+    def __len__(self):
+        if self.results is None:
+            raise RuntimeError("Cannot get length until a slice.")
+        return self.results.hits.total
+
+
+def ElasticsearchPage(*args, **kwargs):  # noqa
+    kwargs.setdefault("wrapper_class", _ElasticsearchWrapper)
+    return Page(*args, **kwargs)
+
+
+def paginate_url_factory(request, query_arg="page"):
+    def make_url(page):
+        query_seq = [
+            (k, v)
+            for k, vs in request.GET.dict_of_lists().items()
+            for v in vs
+            if k != query_arg
+        ]
+        query_seq += [(query_arg, page)]
+        return request.current_route_path(_query=query_seq)
+    return make_url
diff --git a/warehouse/views.py b/warehouse/views.py
@@ -26,6 +26,7 @@
 from warehouse.csrf import csrf_exempt
 from warehouse.packaging.models import Project, Release, File
 from warehouse.sessions import uses_session
+from warehouse.utils.paginate import ElasticsearchPage, paginate_url_factory
 
 
 @view_config(context=HTTPException, decorator=[csrf_exempt])
@@ -105,6 +106,41 @@ def index(request):
     }
 
 
+@view_config(
+    route_name="search",
+    renderer="search/results.html",
+    decorator=[
+        origin_cache(
+            1 * 60 * 60,                      # 1 hour
+            stale_while_revalidate=10 * 60,   # 10 minutes
+            stale_if_error=1 * 24 * 60 * 60,  # 1 day
+            keys=["all-projects"],
+        )
+    ],
+)
+def search(request):
+    if request.params.get("q"):
+        query = request.es.query(
+            "multi_match",
+            query=request.params["q"],
+            fields=[
+                "name", "version", "author", "author_email", "maintainer",
+                "maintainer_email", "home_page", "license", "summary",
+                "description", "keywords", "platform", "download_url",
+            ],
+        )
+    else:
+        query = request.es.query()
+
+    page = ElasticsearchPage(
+        query,
+        page=int(request.params.get("page", 1)),
+        url_maker=paginate_url_factory(request),
+    )
+
+    return {"page": page}
+
+
 @view_config(
     route_name="esi.current-user-indicator",
     renderer="includes/current-user-indicator.html",