Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adding datastore blog samples and tests. #60

Merged
merged 1 commit into from
Jul 29, 2015
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ cache:
directories:
- $HOME/gcloud/
env:
- PATH=$PATH:$HOME/gcloud/google-cloud-sdk/bin GOOGLE_APPLICATION_CREDENTIALS=$TRAVIS_BUILD_DIR/python-docs-samples.json PYTHONPATH=${HOME}/gcloud/google-cloud-sdk/platform/google_appengine TEST_BUCKET_NAME=bigquery-devrel-samples-bucket TEST_PROJECT_ID=bigquery-devrel-samples #Other environment variables on same line
- PATH=$PATH:$HOME/gcloud/google-cloud-sdk/bin GOOGLE_APPLICATION_CREDENTIALS=$TRAVIS_BUILD_DIR/python-docs-samples.json GAE_PYTHONPATH=${HOME}/gcloud/google-cloud-sdk/platform/google_appengine TEST_BUCKET_NAME=bigquery-devrel-samples-bucket TEST_PROJECT_ID=bigquery-devrel-samples #Other environment variables on same line

before_install:
#ENCRYPT YOUR PRIVATE KEY (If you need authentication)
Expand Down Expand Up @@ -35,7 +35,10 @@ before_install:
fi

install:
- pip install tox
- pip install tox coveralls

script:
- tox

after_success:
coveralls
6 changes: 6 additions & 0 deletions appengine/images/tests/test_guestbook.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,12 @@


class TestHandlers(DatastoreTestbedCase):
def setUp(self):
super(TestHandlers, self).setUp()

# Workaround for other tests clobbering our Greeting model.
reload(main)

def test_get(self):
# Build a request object passing the URI path to be tested.
# You can also pass headers, query arguments etc.
Expand Down
5 changes: 5 additions & 0 deletions blog/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# Blog Sample Code

This directory contains samples used in the
[Cloud Platform Blog](http://cloud.google.com/blog). Each sample should have a
readme with instructions and a link to its respective blog post.
22 changes: 22 additions & 0 deletions blog/introduction-to-data-models-in-cloud-datastore/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# Introduction to data models in Cloud Datastore

This sample code is used in [this blog post](). It demonstrates two data models
using [Google Cloud Datstore](https://cloud.google.com/datastore).

## Prerequisites

1. Create project with billing enabled on the [Google Developers Console](https://console.developers.google.com)
2. [Enable the Datastore API](https://console.developers.google.com/project/_/apiui/apiview/datastore/overview).
3. Install the [Google Cloud SDK](https://cloud.google.com/sdk) and be sure to run ``gcloud auth``.


## Running the samples

Install any dependencies:

pip install -r requirements.txt

And run the samples:

python blog.py your-project-id
python wiki.py your-project-id
129 changes: 129 additions & 0 deletions blog/introduction-to-data-models-in-cloud-datastore/blog.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
# Copyright 2015, Google, Inc.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import datetime

from gcloud import datastore


def path_to_key(datastore, path):
"""
Translates a file system path to a datastore key. The basename becomes the
key name and the extension becomes the kind.

Examples:
/file.ext -> key(ext, file)
/parent.ext/file.ext -> key(ext, parent, ext, file)
"""
key_parts = []
path_parts = path.strip(u'/').split(u'/')
for n, x in enumerate(path_parts):
name, ext = x.rsplit('.', 1)
key_parts.extend([ext, name])

return datastore.key(*key_parts)


def create_user(ds, username, profile):
key = path_to_key(ds, '{0}.user'.format(username))
entity = datastore.Entity(key)
entity.update(profile)
ds.put(entity)


def create_post(ds, username, post_content):
now = datetime.datetime.utcnow()
key = path_to_key(ds, '{0}.user/{1}.post'.format(username, now))
entity = datastore.Entity(key)

entity.update({
'created': now,
'created_by': username,
'content': post_content
})

ds.put(entity)


def repost(ds, username, original):
now = datetime.datetime.utcnow()
new_key = path_to_key(ds, '{0}.user/{1}.post'.format(username, now))
new = datastore.Entity(new_key)

new.update(original)

ds.put(new)


def list_posts_by_user(ds, username):
user_key = path_to_key(ds, '{0}.user'.format(username))
return ds.query(kind='post', ancestor=user_key).fetch()


def list_all_posts(ds):
return ds.query(kind='post').fetch()


def main(project_id):
ds = datastore.Client(dataset_id=project_id)

print("Creating users...")
create_user(ds, 'tonystark',
{'name': 'Tony Stark', 'location': 'Stark Island'})
create_user(ds, 'peterparker',
{'name': 'Peter Parker', 'location': 'New York City'})

print("Creating posts...")
for n in range(1, 10):
create_post(ds, 'tonystark', "Tony's post #{0}".format(n))
create_post(ds, 'peterparker', "Peter's post #{0}".format(n))

print("Re-posting tony's post as peter...")

tonysposts = list_posts_by_user(ds, 'tonystark')
for post in tonysposts:
original_post = post
break

repost(ds, 'peterparker', original_post)

print('Posts by tonystark:')
for post in list_posts_by_user(ds, 'tonystark'):
print("> {0} on {1}".format(post['content'], post['created']))

print('Posts by peterparker:')
for post in list_posts_by_user(ds, 'peterparker'):
print("> {0} on {1}".format(post['content'], post['created']))

print('Posts by everyone:')
for post in list_all_posts(ds):
print("> {0} on {1}".format(post['content'], post['created']))

print('Cleaning up...')
ds.delete_multi([
path_to_key(ds, 'tonystark.user'),
path_to_key(ds, 'peterparker.user')
])
ds.delete_multi([
x.key for x in list_all_posts(ds)])


if __name__ == "__main__":
parser = argparse.ArgumentParser(
description='Demonstrates wiki data model.')
parser.add_argument('project_id', help='Your cloud project ID.')

args = parser.parse_args()

main(args.project_id)
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
gcloud==0.7.0
google-apitools==0.4.8
httplib2==0.9.1
oauth2client==1.4.12
protobuf==3.0.0a1
protorpc==0.10.0
pyasn1==0.1.8
pyasn1-modules==0.0.6
pycrypto==2.6.1
pytz==2015.4
rsa==3.1.4
six==1.9.0
22 changes: 22 additions & 0 deletions blog/introduction-to-data-models-in-cloud-datastore/test_blog.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# Copyright 2015, Google, Inc.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from blog import main
from tests import CloudBaseTest


class BlogTestCase(CloudBaseTest):
"""Simple test case that ensures the blog code doesn't throw any errors."""

def test_main(self):
main(self.constants['projectId'])
107 changes: 107 additions & 0 deletions blog/introduction-to-data-models-in-cloud-datastore/wiki.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
# Copyright 2015, Google, Inc.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import datetime

from gcloud import datastore


def path_to_key(datastore, path):
"""
Translates a file system path to a datastore key. The basename becomes the
key name and the extension becomes the kind.

Examples:
/file.ext -> key(ext, file)
/parent.ext/file.ext -> key(ext, parent, ext, file)
"""
key_parts = []
path_parts = path.strip(u'/').split(u'/')
for n, x in enumerate(path_parts):
name, ext = x.rsplit('.', 1)
key_parts.extend([ext, name])

return datastore.key(*key_parts)


def save_page(ds, page, content):
with ds.transaction():
now = datetime.datetime.utcnow()
current_key = path_to_key(ds, '{}.page/current.revision'.format(page))
revision_key = path_to_key(ds, '{}.page/{}.revision'.format(page, now))

if ds.get(revision_key):
raise AssertionError("Revision %s already exists" % revision_key)

current = ds.get(current_key)

if current:
revision = datastore.Entity(key=revision_key)
revision.update(current)
ds.put(revision)
else:
current = datastore.Entity(key=current_key)

current['content'] = content

ds.put(current)


def restore_revision(ds, page, revision):
save_page(ds, page, revision['content'])


def list_pages(ds):
return ds.query(kind='page').fetch()


def list_revisions(ds, page):
page_key = path_to_key(ds, '{}.page'.format(page))
return ds.query(kind='revision', ancestor=page_key).fetch()


def main(project_id):
ds = datastore.Client(dataset_id=project_id)

save_page(ds, 'page1', '1')
save_page(ds, 'page1', '2')
save_page(ds, 'page1', '3')

print('Revisions for page1:')
first_revision = None
for revision in list_revisions(ds, 'page1'):
if not first_revision:
first_revision = revision
print("{}: {}".format(revision.key.name, revision['content']))

print('restoring revision {}:'.format(first_revision.key.name))
restore_revision(ds, 'page1', first_revision)

print('Revisions for page1:')
for revision in list_revisions(ds, 'page1'):
print("{}: {}".format(revision.key.name, revision['content']))

print('Cleaning up')
ds.delete_multi([path_to_key(ds, 'page1.page')])
ds.delete_multi([x.key for x in list_revisions(ds, 'page1')])


if __name__ == "__main__":
parser = argparse.ArgumentParser(
description='Demonstrates wiki data model.')
parser.add_argument('project_id', help='Your cloud project ID.')

args = parser.parse_args()

main(args.project_id)
21 changes: 17 additions & 4 deletions tests/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,17 @@
import os
import StringIO
import sys
import tempfile
import unittest

from google.appengine.datastore import datastore_stub_util
from google.appengine.ext import testbed
from nose.plugins.skip import SkipTest

try:
APPENGINE_AVAILABLE = True
from google.appengine.datastore import datastore_stub_util
from google.appengine.ext import testbed
except ImportError:
APPENGINE_AVAILABLE = False

BUCKET_NAME_ENV = 'TEST_BUCKET_NAME'
PROJECT_ID_ENV = 'TEST_PROJECT_ID'
Expand Down Expand Up @@ -82,12 +89,16 @@ def setUp(self):
self.constants['cloudStorageOutputURI'] % test_bucket_name)

def tearDown(self):
os.environ['SERVER_SOFTWARE'] = self._server_software_org
if self._server_software_org:
os.environ['SERVER_SOFTWARE'] = self._server_software_org


class DatastoreTestbedCase(unittest.TestCase):
"""A base test case for common setup/teardown tasks for test."""
def setUp(self):
if not APPENGINE_AVAILABLE:
raise SkipTest()

"""Setup the datastore and memcache stub."""
# First, create an instance of the Testbed class.
self.testbed = testbed.Testbed()
Expand All @@ -99,7 +110,9 @@ def setUp(self):
self.policy = datastore_stub_util.PseudoRandomHRConsistencyPolicy(
probability=0)
# Initialize the datastore stub with this policy.
self.testbed.init_datastore_v3_stub(consistency_policy=self.policy)
self.testbed.init_datastore_v3_stub(
datastore_file=tempfile.mkstemp()[1],
consistency_policy=self.policy)
self.testbed.init_memcache_stub()

def tearDown(self):
Expand Down
Loading