From a64a530b4a261813596d9c17029fd184121e0af9 Mon Sep 17 00:00:00 2001 From: Stanislaw Szydlo <stanislaw.szydlo@ericpol.com> Date: Thu, 22 Oct 2015 11:33:00 +0200 Subject: [PATCH] [firstapp] Implementation of section2 for openstacksdk Change-Id: Ib893f353b2567b6416a117e694f2d4aee692ca3a Partial-Bug: 1449328 --- firstapp/samples/openstacksdk/introduction.py | 228 ++++++++++++++++++ firstapp/source/introduction.rst | 163 ++++++++++--- 2 files changed, 352 insertions(+), 39 deletions(-) create mode 100644 firstapp/samples/openstacksdk/introduction.py diff --git a/firstapp/samples/openstacksdk/introduction.py b/firstapp/samples/openstacksdk/introduction.py new file mode 100644 index 000000000..67aebfe02 --- /dev/null +++ b/firstapp/samples/openstacksdk/introduction.py @@ -0,0 +1,228 @@ +# step-1 +userdata = '''#!/usr/bin/env bash +curl -L -s https://git.openstack.org/cgit/openstack/faafo/plain/contrib/install.sh | bash -s -- \ + -i faafo -i messaging -r api -r worker -r demo +''' +userdata_b64str = base64.b64encode(userdata) + +instance_name = 'all-in-one' +testing_instance_args = { + 'name': instance_name, + 'imageRef': image, + 'flavorRef': flavor, + 'key_name': keypair_name, + 'user_data': userdata_b64str, + 'security_groups': [{'name': all_in_one_security_group.name}] +} + +testing_instance = conn.compute.create_server(**testing_instance_args) + +# step-2 +userdata = '''#!/usr/bin/env bash +curl -L -s https://git.openstack.org/cgit/openstack/faafo/plain/contrib/install.sh | bash -s -- \ + -i faafo -i messaging -r api -r worker -r demo +''' +userdata_b64str = base64.b64encode(userdata) + +# step-3 +security_group_args = { + 'name' : 'all-in-one', + 'description': 'network access for all-in-one application.' +} +all_in_one_security_group = conn.network.create_security_group(**security_group_args) + +# HTTP access +security_rule_args = { + 'security_group_id': all_in_one_security_group, + 'direction': 'ingress', + 'protocol': 'tcp', + 'port_range_min': '80', + 'port_range_max': '80' +} +conn.network.create_security_group_rule(**security_rule_args) + +# SSH access +security_rule_args['port_range_min'] = '22' +security_rule_args['port_range_max'] = '22' +conn.network.create_security_group_rule(**security_rule_args) + +# step-4 +for security_group in conn.network.security_groups(): + print(security_group) + +# step-5 +conn.network.delete_security_group_rule(rule) +conn.network.delete_security_group(security_group) + +# step-6 +testing_instance['security_groups'] + +# step-7 +unused_floating_ip = None +for floating_ip in conn.network.ips(): + if not floating_ip.fixed_ip_address: + unused_floating_ip = floating_ip + print("Found an unused Floating IP: %s" % floating_ip) + break + +# step-8 +public_network_id = conn.network.find_network('public').id + +# step-9 +unused_floating_ip = conn.network.create_ip(floating_network_id=public_network_id) +unused_floating_ip = conn.network.get_ip(unused_floating_ip) + +# step-10 +for port in conn.network.ports(): + if port.device_id == testing_instance.id: + testing_instance_port = port + break + +testing_instance_floating_ip = unused_floating_ip +conn.network.add_ip_to_port(testing_instance_port, testing_instance_floating_ip) + +# step-11 +security_group_args = { + 'name' : 'worker', + 'description': 'for services that run on a worker node' +} +worker_group = conn.network.create_security_group(**security_group_args) + +security_rule_args = { + 'security_group_id': worker_group, + 'direction': 'ingress', + 'protocol': 'tcp', + 'port_range_min': '22', + 'port_range_max': '22' +} +conn.network.create_security_group_rule(**security_rule_args) + +security_group_args = { + 'name' : 'control', + 'description': 'for services that run on a control node' +} +controller_group = conn.network.create_security_group(**security_group_args) + +# Switch to controller_group and readd SSH access rule +security_rule_args['security_group_id'] = controller_group +conn.network.create_security_group_rule(**security_rule_args) + +# Add HTTP access rule +security_rule_args['port_range_min'] = '80' +security_rule_args['port_range_max'] = '80' +conn.network.create_security_group_rule(**security_rule_args) + +# Add RabbitMQ access rule for all instances with +# 'worker' security group +security_rule_args['port_range_min'] = '5672' +security_rule_args['port_range_max'] = '5672' +security_rule_args['remote_group_id'] = worker_group +conn.network.create_security_group_rule(**security_rule_args) + +userdata = '''#!/usr/bin/env bash +curl -L -s http://git.openstack.org/cgit/openstack/faafo/plain/contrib/install.sh | bash -s -- \ + -i messaging -i faafo -r api +''' +userdata_b64str = base64.b64encode(userdata) + +instance_controller_1_args = { + 'name': 'app-controller', + 'imageRef': image, + 'flavorRef': flavor, + 'key_name': 'demokey', + 'user_data': userdata_b64str, + 'security_groups': [{'name': controller_group.name}] +} + +instance_controller_1 = conn.compute.create_server(**instance_controller_1_args) +conn.compute.wait_for_server(instance_controller_1) + +print('Checking for unused Floating IP...') +unused_floating_ip = None +for floating_ip in conn.network.ips(): + if not floating_ip.fixed_ip_address: + unused_floating_ip = floating_ip + print("Found an unused Floating IP: %s" % floating_ip) + break + +if not unused_floating_ip: + print('No free unused Floating IPs. Allocating new Floating IP...') + public_network_id = conn.network.find_network('public').id + unused_floating_ip = conn.network.create_ip(floating_network_id=public_network_id) + unused_floating_ip = conn.network.get_ip(unused_floating_ip) + +for port in conn.network.ports(): + if port.device_id == instance_controller_1.id: + controller_instance_port = port + break + +controller_instance_floating_ip = unused_floating_ip +conn.network.add_ip_to_port(controller_instance_port, controller_instance_floating_ip) + +# Retrieve all information about 'instance_controller_1' +instance_controller_1 = conn.compute.get_server(instance_controller_1) + +print('Application will be deployed to http://%s' % controller_instance_floating_ip.floating_ip_address) + +# step-12 +for values in instance_controller_1.addresses.itervalues(): + for address in values: + if address['OS-EXT-IPS:type'] == 'fixed': + ip_controller = address['addr'] + break + +userdata = '''#!/usr/bin/env bash +curl -L -s http://git.openstack.org/cgit/openstack/faafo/plain/contrib/install.sh | bash -s -- \ + -i faafo -r worker -e 'http://%(ip_controller)s' -m 'amqp://guest:guest@%(ip_controller)s:5672/' +''' % {'ip_controller': ip_controller} +userdata_b64str = base64.b64encode(userdata) + +instance_worker_1_args = { + 'name': 'app-worker-1', + 'imageRef': image, + 'flavorRef': flavor, + 'key_name': 'demokey', + 'user_data': userdata_b64str, + 'security_groups': [{'name': worker_group.name}] +} + +instance_worker_1 = conn.compute.create_server(**instance_worker_1_args) +conn.compute.wait_for_server(instance_worker_1) + +print('Checking for unused Floating IP...') +unused_floating_ip = None +for floating_ip in conn.network.ips(): + if not floating_ip.fixed_ip_address: + unused_floating_ip = floating_ip + print("Found an unused Floating IP: %s" % floating_ip) + break + +if not unused_floating_ip: + print('No free unused Floating IPs. Allocating new Floating IP...') + public_network_id = conn.network.find_network('public').id + unused_floating_ip = conn.network.create_ip(floating_network_id=public_network_id) + unused_floating_ip = conn.network.get_ip(unused_floating_ip) + +for port in conn.network.ports(): + if port.device_id == instance_worker_1.id: + worker_instance_port = port + break + +worker_instance_floating_ip = unused_floating_ip +conn.network.add_ip_to_port(worker_instance_port, worker_instance_floating_ip) + +# Retrieve all information about 'instance_worker_1' +instance_worker_1 = conn.compute.get_server(instance_worker_1) + +print('The worker will be available for SSH at %s' % worker_instance_floating_ip.floating_ip_address) + +# step-13 +for values in instance_worker_1.addresses.itervalues(): + for address in values: + if address['OS-EXT-IPS:type'] == 'floating': + ip_instance_worker_1 = address['addr'] + break + +print(ip_instance_worker_1) + +# step-14 diff --git a/firstapp/source/introduction.rst b/firstapp/source/introduction.rst index 98ecbb828..b0bb927a2 100644 --- a/firstapp/source/introduction.rst +++ b/firstapp/source/introduction.rst @@ -24,10 +24,6 @@ particular. It also describes some commands in the previous section. .. warning:: This section has not yet been completed for the pkgcloud SDK. -.. only:: openstacksdk - - .. warning:: This section has not yet been completed for the OpenStack SDK. - .. only:: phpopencloud .. warning:: This section has not yet been completed for the @@ -61,20 +57,20 @@ Cloud applications often use many small instances rather than a few large instances. Provided that an application is sufficiently modular, you can easily distribute micro-services across as many instances as required. This architecture enables an application to grow past the limit imposed by the -maximum size of an instance. It's like trying to move a large number of people -from one place to another; there's only so many people you can put on the +maximum size of an instance. It is like trying to move a large number of people +from one place to another; there is only so many people you can put on the largest bus, but you can use an unlimited number of buses or small cars, which provide just the capacity you need - and no more. Fault tolerance --------------- -In cloud programming, there's a well-known analogy known as "cattle vs -pets". If you haven't heard it before, it goes like this: +In cloud programming, there is a well-known analogy known as "cattle vs +pets". If you have not heard it before, it goes like this: -When you're dealing with pets, you name them and care for them and if +When you are dealing with pets, you name them and care for them and if they get sick, you nurse them back to health. Nursing pets back to -health can be difficult and very time consuming. When you're dealing +health can be difficult and very time consuming. When you are dealing with cattle, you attach a numbered tag to their ear and if they get sick you put them down and move on. @@ -85,24 +81,24 @@ went wrong with one of those servers, the staff's job was to do whatever it took to make it right again and save the server and the application. -In cloud programming, it's very different. Rather than large, -expensive servers, you're dealing with virtual machines that are +In cloud programming, it is very different. Rather than large, +expensive servers, you are dealing with virtual machines that are literally disposable; if something goes wrong, you shut it down and -spin up a new one. There's still operations staff, but rather than +spin up a new one. There is still operations staff, but rather than nursing individual servers back to health, their job is to monitor the health of the overall system. -There are definite advantages to this architecture. It's easy to get a +There are definite advantages to this architecture. It is easy to get a "new" server, without any of the issues that inevitably arise when a server has been up and running for months, or even years. As with classical infrastructure, failures of the underpinning cloud infrastructure (hardware, networks, and software) are -unavoidable. When you're designing for the cloud, it's crucial that +unavoidable. When you are designing for the cloud, it is crucial that your application is designed for an environment where failures can -happen at any moment. This may sound like a liability, but it's not; +happen at any moment. This may sound like a liability, but it is not; by designing your application with a high degree of fault tolerance, -you're also making it resilient in the face of change, and therefore +you are also making it resilient in the face of change, and therefore more adaptable. Fault tolerance is essential to the cloud-based application. @@ -130,9 +126,9 @@ Fractals application architecture ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The Fractals application was designed with the principles of the previous -subsection in mind. You'll note that in :doc:`getting_started`, we deployed the -application in an all-in-one style, on a single virtual machine. This isn't -good practice, but because the application uses micro-services to decouple +subsection in mind. You will note that in :doc:`getting_started`, we deployed the +application in an all-in-one style, on a single virtual machine. This is not +a good practice, but because the application uses micro-services to decouple logical application functions, we can change this easily. .. graphviz:: images/architecture.dot @@ -162,7 +158,7 @@ worker may crash and the tasks will be processed by other workers. The worker service consumes messages from the work queue and then processes them to create the corresponding fractal image file. -Of course there's also a web interface which offers a more human +Of course there is also a web interface which offers a more human friendly way of accessing the API to view the created fractal images, and a simple command line interface. @@ -176,7 +172,7 @@ and a simple command line interface. There are also multiple storage back ends (to store the generated fractal images) and a database component (to store the state of -tasks), but we'll talk about those in :doc:`/durability` and +tasks), but we will talk about those in :doc:`/durability` and :doc:`/block_storage` respectively. How the Fractals application interacts with OpenStack @@ -188,7 +184,7 @@ How the Fractals application interacts with OpenStack across each section. Adding it here forces the introduction of block storage, object storage, orchestration and neutron networking too early, which could seriously - confuse users who don't have these services in their + confuse users who do not have these services in their cloud. Therefore, this should not be done here. @@ -196,9 +192,9 @@ The magic revisited ~~~~~~~~~~~~~~~~~~~ So what exactly was that request doing at the end of the previous -section? Let's look at it again. (Note that in this subsection, we're -just explaining what you've already done in the previous section; you -don't need to execute these commands again.) +section? Let us look at it again. (Note that in this subsection, we are +just explaining what you have already done in the previous section; you +do not need to execute these commands again.) .. only:: shade @@ -213,10 +209,21 @@ don't need to execute these commands again.) :start-after: step-1 :end-before: step-2 +.. only:: openstacksdk + + .. literalinclude:: ../samples/openstacksdk/introduction.py + :start-after: step-1 + :end-before: step-2 + We explained image and flavor in :doc:`getting_started`, so in the following sections, we will explain the other parameters in detail, including :code:`ex_userdata` (cloud-init) and :code:`ex_keyname` (key pairs). +.. only:: openstacksdk + + .. note:: In openstacksdk parameter :code:`ex_userdata` is called :code:`user_data` + and parameter :code:`ex_keyname` is called :code:`key_name`. + Introduction to cloud-init -------------------------- @@ -252,6 +259,13 @@ your cloud provider to confirm the user name. :start-after: step-2 :end-before: step-3 +.. only:: openstacksdk + + .. literalinclude:: ../samples/openstacksdk/introduction.py + :start-after: step-2 + :end-before: step-3 + + .. note:: User data in openstacksdk must be encoded to base64. After the instance is created, cloud-init downloads and runs a script called :code:`install.sh`. This script installs the Fractals application. Cloud-init @@ -262,7 +276,7 @@ about cloud-init in the `official documentation <https://cloudinit.readthedocs.o Introduction to key pairs ------------------------- -Security is important when it comes to your instances; you can't have just +Security is important when it comes to your instances; you can not have just anyone accessing them. To enable logging into an instance, you must provide the public key of an SSH key pair during instance creation. In section one, you created and uploaded a key pair to OpenStack, and cloud-init installed it @@ -303,6 +317,12 @@ port 22): ports as input. This is why ports 80 and 22 are passed twice. +.. only:: openstacksdk + + .. literalinclude:: ../samples/openstacksdk/introduction.py + :start-after: step-3 + :end-before: step-4 + You can list available security groups with: .. only:: shade @@ -318,8 +338,13 @@ You can list available security groups with: :start-after: step-4 :end-before: step-5 +.. only:: openstacksdk -Once you've created a rule or group, you can also delete it: + .. literalinclude:: ../samples/openstacksdk/introduction.py + :start-after: step-4 + :end-before: step-5 + +Once you have created a rule or group, you can also delete it: .. only:: shade @@ -334,6 +359,11 @@ Once you've created a rule or group, you can also delete it: :start-after: step-5 :end-before: step-6 +.. only:: openstacksdk + + .. literalinclude:: ../samples/openstacksdk/introduction.py + :start-after: step-5 + :end-before: step-6 To see which security groups apply to an instance, you can: @@ -350,10 +380,15 @@ To see which security groups apply to an instance, you can: :start-after: step-6 :end-before: step-7 +.. only:: openstacksdk + + .. literalinclude:: ../samples/openstacksdk/introduction.py + :start-after: step-6 + :end-before: step-7 .. todo:: print() ? -Once you've configured permissions, you'll need to know where to +Once you have configured permissions, you will need to know where to access the application. Introduction to Floating IPs @@ -412,6 +447,27 @@ then associate it to your instance's network interface. :start-after: step-7 :end-before: step-8 +.. only:: openstacksdk + + .. literalinclude:: ../samples/openstacksdk/introduction.py + :start-after: step-7 + :end-before: step-8 + + If you have no free floating IPs that have been allocated for + your project, first select a network which offer allocation + of floating IPs. In this example we use network which is + called :code:`public`. + + .. literalinclude:: ../samples/openstacksdk/introduction.py + :start-after: step-8 + :end-before: step-9 + + Now request an address from this network to be allocated to your project. + + .. literalinclude:: ../samples/openstacksdk/introduction.py + :start-after: step-9 + :end-before: step-10 + Now that you have an unused floating IP address allocated to your project, attach it to an instance. @@ -428,22 +484,28 @@ project, attach it to an instance. :start-after: step-10 :end-before: step-11 +.. only:: openstacksdk + + .. literalinclude:: ../samples/openstacksdk/introduction.py + :start-after: step-10 + :end-before: step-11 + That brings us to where we ended up at the end of :doc:`/getting_started`. But where do we go from here? Splitting services across multiple instances ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -We've talked about separating functions into different micro-services, +We have talked about separating functions into different micro-services, and how that enables us to make use of the cloud architecture. Now -let's see that in action. +let us see that in action. -The rest of this tutorial won't reference the all-in-one instance you +The rest of this tutorial will not reference the all-in-one instance you created in section one. Take a moment to delete this instance. -It's easy to split out services into multiple instances. We will +It is easy to split out services into multiple instances. We will create a controller instance called :code:`app-controller`, which -hosts the API, database, and messaging services. We'll also create a +hosts the API, database, and messaging services. We will also create a worker instance called :code:`app-worker-1`, which just generates fractals. @@ -474,7 +536,13 @@ Parameter Description Values :start-after: step-11 :end-before: step-12 -Note that this time, when you create a security group, you're +.. only:: openstacksdk + + .. literalinclude:: ../samples/openstacksdk/introduction.py + :start-after: step-11 + :end-before: step-12 + +Note that this time, when you create a security group, you are including a rule that only applies for instances that are part of the worker_group. @@ -495,11 +563,17 @@ Next, start a second instance, which will be the worker instance: :start-after: step-12 :end-before: step-13 -Notice that you've added this instance to the worker_group, so it can +.. only:: openstacksdk + + .. literalinclude:: ../samples/openstacksdk/introduction.py + :start-after: step-12 + :end-before: step-13 + +Notice that you have added this instance to the worker_group, so it can access the controller. As you can see from the parameters passed to the installation script, you are -specifying that this is the worker instance, but you're also passing the +specifying that this is the worker instance, but you are also passing the address of the API instance and the message queue so the worker can pick up requests. The Fractals application installation script can take several parameters. @@ -537,6 +611,12 @@ address of the worker: :start-after: step-13 :end-before: step-14 +.. only:: openstacksdk + + .. literalinclude:: ../samples/openstacksdk/introduction.py + :start-after: step-13 + :end-before: step-14 + Now you can SSH into the instance: :: @@ -546,7 +626,7 @@ Now you can SSH into the instance: .. note:: Replace :code:`IP_WORKER_1` with the IP address of the worker instance and USERNAME to the appropriate user name. -Once you've logged in, check to see whether the worker service process +Once you have logged in, check to see whether the worker service process is running as expected. You can find the logs of the worker service in the directory :code:`/var/log/supervisor/`. @@ -621,7 +701,7 @@ with :code:`faafo get --help`, :code:`faafo list --help`, and .. note:: The application stores the generated fractal images directly in the database used by the API service instance. Storing - image files in a database is not good practice. We're doing it + image files in a database is not good practice. We are doing it here as an example only as an easy way to allow multiple instances to have access to the data. For best practice, we recommend storing objects in Object Storage, which is @@ -669,3 +749,8 @@ information, the flavor ID, and image ID. .. literalinclude:: ../samples/libcloud/introduction.py :language: python + +.. only:: openstacksdk + + .. literalinclude:: ../samples/openstacksdk/introduction.py + :language: python