From 4f58bc6b17b93beca19746f3c8141f5414978ffb Mon Sep 17 00:00:00 2001 From: Aleksei Krugliak Date: Sun, 1 Sep 2024 15:19:36 +0400 Subject: [PATCH] add bucket and private-cluster-module --- bucket/README.md | 81 +++++++++++ bucket/main.tf | 18 +++ bucket/outputs.tf | 14 ++ bucket/providers.tf | 6 + bucket/variables.tf | 17 +++ bucket/versions.tf | 11 ++ private-cluster-module/README.md | 160 ++++++++++++++++++++++ private-cluster-module/backend.tf | 10 ++ private-cluster-module/demo-namespace.tf | 6 + private-cluster-module/get-credentials.sh | 9 ++ private-cluster-module/gke.tf | 73 ++++++++++ private-cluster-module/outputs.tf | 45 ++++++ private-cluster-module/providers.tf | 12 ++ private-cluster-module/variables.tf | 63 +++++++++ private-cluster-module/versions.tf | 21 +++ private-cluster-module/vpc.tf | 27 ++++ 16 files changed, 573 insertions(+) create mode 100644 bucket/README.md create mode 100644 bucket/main.tf create mode 100644 bucket/outputs.tf create mode 100644 bucket/providers.tf create mode 100644 bucket/variables.tf create mode 100644 bucket/versions.tf create mode 100644 private-cluster-module/README.md create mode 100644 private-cluster-module/backend.tf create mode 100644 private-cluster-module/demo-namespace.tf create mode 100755 private-cluster-module/get-credentials.sh create mode 100644 private-cluster-module/gke.tf create mode 100644 private-cluster-module/outputs.tf create mode 100644 private-cluster-module/providers.tf create mode 100644 private-cluster-module/variables.tf create mode 100644 private-cluster-module/versions.tf create mode 100644 private-cluster-module/vpc.tf diff --git a/bucket/README.md b/bucket/README.md new file mode 100644 index 0000000..629c9c9 --- /dev/null +++ b/bucket/README.md @@ -0,0 +1,81 @@ +# bucket + +You should be an admin/owner of your sandbox or have enougth permissions to create a bucket. + +If you use a lot of accounts and specific IAM rules, you can check the permissions of the user with the following command: + +```shell +gcloud projects get-iam-policy --format=json | jq '.bindings[] | select(.members[] | contains("user:"))' +``` + +`"role": "roles/owner"` would be great for you. + +## How to use the code + +1. Create `terraform.tfvars` file with a few variables + +```shell +project = "your-gcp-project" +iam_user_email = "your@gmail.com" +``` + +`region` is optional. + +2. Create bucket + +All commands will be applied via Terraform 1.8.0 or via OpenTofu, the same version. +I use alias `t` for the commands. + +```shell +t init +t apply +``` + +2. Destrouy bucket sfter your tests + +```shell +t destroy +``` + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | ~>1.8.0 | +| [google](#requirement\_google) | ~>5.41.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | 5.41.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [bucket](#module\_bucket) | terraform-google-modules/cloud-storage/google//modules/simple_bucket | ~> 6.0 | + +## Resources + +| Name | Type | +|------|------| +| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [iam\_user\_email](#input\_iam\_user\_email) | Your gcloud account | `string` | `"demo@gmail.com"` | no | +| [project](#input\_project) | Google Project to create resources in | `string` | `"demo"` | no | +| [region](#input\_region) | The region to host the cluster in | `string` | `"europe-west1"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [gcloud\_bucket\_link](#output\_gcloud\_bucket\_link) | Bucket web ui link | +| [project](#output\_project) | GCloud Project ID | +| [region](#output\_region) | GCloud Region | + \ No newline at end of file diff --git a/bucket/main.tf b/bucket/main.tf new file mode 100644 index 0000000..e00fb66 --- /dev/null +++ b/bucket/main.tf @@ -0,0 +1,18 @@ +module "bucket" { + # https://registry.terraform.io/modules/terraform-google-modules/cloud-storage/google/6.0.1/submodules/simple_bucket?tab=inputs + source = "terraform-google-modules/cloud-storage/google//modules/simple_bucket" + version = "~> 6.0" + + name = "${var.project}-bucket" + project_id = var.project + location = "eu" + + force_destroy = true # Use this only for testing purposes + + iam_members = [{ + role = "roles/storage.objectAdmin" + member = "user:${var.iam_user_email}" + }] + + autoclass = true +} diff --git a/bucket/outputs.tf b/bucket/outputs.tf new file mode 100644 index 0000000..8bd27d2 --- /dev/null +++ b/bucket/outputs.tf @@ -0,0 +1,14 @@ +output "region" { + description = "GCloud Region" + value = var.region +} + +output "project" { + description = "GCloud Project ID" + value = var.project +} + +output "gcloud_bucket_link" { + description = "Bucket web ui link" + value = "https://console.cloud.google.com/storage/browser/${module.bucket.name};tab=objects?forceOnBucketsSortingFiltering=true&project=${var.project}&prefix=&forceOnObjectsSortingFiltering=false" +} diff --git a/bucket/providers.tf b/bucket/providers.tf new file mode 100644 index 0000000..6a6caeb --- /dev/null +++ b/bucket/providers.tf @@ -0,0 +1,6 @@ +data "google_client_config" "default" {} + +provider "google" { + project = var.project + region = var.region +} diff --git a/bucket/variables.tf b/bucket/variables.tf new file mode 100644 index 0000000..fd161dc --- /dev/null +++ b/bucket/variables.tf @@ -0,0 +1,17 @@ +variable "project" { + type = string + description = "Google Project to create resources in" + default = "demo" +} + +variable "region" { + type = string + description = "The region to host the cluster in" + default = "europe-west1" +} + +variable "iam_user_email" { + type = string + description = "Your gcloud account" + default = "demo@gmail.com" +} diff --git a/bucket/versions.tf b/bucket/versions.tf new file mode 100644 index 0000000..018d130 --- /dev/null +++ b/bucket/versions.tf @@ -0,0 +1,11 @@ +terraform { + required_version = "~>1.8.0" + + required_providers { + # https://github.com/hashicorp/terraform-provider-google + google = { + source = "hashicorp/google" + version = "~>5.41.0" + } + } +} diff --git a/private-cluster-module/README.md b/private-cluster-module/README.md new file mode 100644 index 0000000..8249cae --- /dev/null +++ b/private-cluster-module/README.md @@ -0,0 +1,160 @@ +# private-cluster-module + +## What is created ty thah template + +This example is using remote bucket state. You can modify this manually or not. + +1. VPC network with one subnet +2. GKE cluster with external endpoint and only authorized networks access. +3. Workload nodepool with one node +3. Namespace `this-is-demo-cluster` + +## How to use the code + +### Optional bucket usage. +If you would try to use bucket state, you need to uncomment code block in [backend.tf](backend.tf). + +Then go to [../bucket](../bucket/) directory and create the bucket by following the instructions at [README](../bucket/README.md). + +Then return to this directory. + +1. Create `terraform.tfvars` file with a few variables + +```shell +project = "your-gcp-project" +host_project = "your-gcp-project" + +master_authorized_networks = [ + { cidr_block = "0.0.0.1/32", display_name = "your current IP" } +] +``` + +`region`, `zone` and `environment_name` are optional + +Your current IP you can get via this command for example: + +```shell +curl -s ifconfig.me +``` + +2. Create cluster +All commands will be applied via Terraform 1.8.0 or via OpenTofu, the same version. +I use alias `t` for the commands. + +```shell +t init +t apply +``` + +3. Get the credentials for the new cluster (configure kubeconfig) + +You can see all useful commands and links in the output: + +```shell +t output +``` + +There is a manual command: + +```shell +gcloud container clusters get-credentials $(t output -raw kubernetes_cluster_name) --region $(t output -raw zone) --project $(t output -raw project) +``` + +Or just use `./get-credentials.sh` + +4. Destroy all resources + +```shell +t destroy +``` + +## Additional info + +Some manual tests. + +### terraform +✅ create cluster
+✅ `./get-credentials.sh`
+✅ try access from different networks
+✅ manual cred command
+✅ output `gcloud_gke_get_creds` command
+✅ output links
+✅ create simple nginx pod `kubectl run nginx --image=nginx:latest -n default && kubectl get po -w`
+✅ `flux install && kubectl get po -n flux-system -w`
+✅ destroy cluster
+✅ test bucket backend + +### opentofu +✅ create cluster
+✅ `./get-credentials.sh`
+✅ try access from different networks
+✅ manual cred command
+✅ output `gcloud_gke_get_creds` command
+✅ output links
+✅ create simple nginx pod `kubectl run nginx --image=nginx:latest -n default && kubectl get po -w`
+✅ `flux install && kubectl get po -n flux-system -w`
+✅ destroy cluster
+✅ test bucket backend + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | ~>1.8.0 | +| [google](#requirement\_google) | ~>5.41.0 | +| [google-beta](#requirement\_google-beta) | ~> 5.41 | +| [kubernetes](#requirement\_kubernetes) | ~>2.32.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | 5.41.0 | +| [kubernetes](#provider\_kubernetes) | 2.32.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [gke](#module\_gke) | terraform-google-modules/kubernetes-engine/google//modules/beta-private-cluster-update-variant | 32.0.0 | + +## Resources + +| Name | Type | +|------|------| +| [google_compute_network.vpc](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_network) | resource | +| [google_compute_subnetwork.subnet](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_subnetwork) | resource | +| [google_project_service.service_networking](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_service) | resource | +| [kubernetes_namespace.this-is-demo-cluster](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace) | resource | +| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [bucket\_credentials\_json](#input\_bucket\_credentials\_json) | Default path to your gcloud credentials json file | `string` | `"~/.config/gcloud/application_default_credentials.json"` | no | +| [bucket\_name](#input\_bucket\_name) | Bucket name | `string` | `"your-project-bucket"` | no | +| [bucket\_prefix](#input\_bucket\_prefix) | Path to store your state in bucket | `string` | `"terraform/state/demo"` | no | +| [environment\_name](#input\_environment\_name) | Environment name | `string` | `"demo"` | no | +| [host\_project](#input\_host\_project) | The GCP project housing the VPC network to host the cluster in | `string` | `"demo"` | no | +| [master\_authorized\_networks](#input\_master\_authorized\_networks) | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` |
[
{
"cidr_block": "1.1.1.1/32",
"display_name": "Use your IP/VPN IP here"
}
]
| no | +| [project](#input\_project) | Google Project to create resources in | `string` | `"demo"` | no | +| [region](#input\_region) | The region to host the cluster in | `string` | `"europe-west1"` | no | +| [vpc\_network](#input\_vpc\_network) | The GCP network to apply firewall rules in | `string` | `"demo-vpc"` | no | +| [zone](#input\_zone) | The region to host the cluster in | `string` | `"europe-west1-b"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [gcloud\_gke\_get\_creds](#output\_gcloud\_gke\_get\_creds) | Command to get GKE credentials | +| [gcloud\_gke\_link](#output\_gcloud\_gke\_link) | GKE web ui link | +| [gcloud\_vpc\_link](#output\_gcloud\_vpc\_link) | VPC web ui link | +| [kubernetes\_cluster\_host](#output\_kubernetes\_cluster\_host) | GKE Cluster Host | +| [kubernetes\_cluster\_name](#output\_kubernetes\_cluster\_name) | GKE Cluster Name | +| [master\_authorized\_networks\_config](#output\_master\_authorized\_networks\_config) | Here are networks that are allowed to reach your k8s API | +| [project](#output\_project) | GCloud Project ID | +| [region](#output\_region) | GCloud Region | +| [zone](#output\_zone) | GCloud Project ID | + diff --git a/private-cluster-module/backend.tf b/private-cluster-module/backend.tf new file mode 100644 index 0000000..34bef66 --- /dev/null +++ b/private-cluster-module/backend.tf @@ -0,0 +1,10 @@ +# Optionallly you can use bucket as backend +# If you would to do that - uncomment code below and follow instructions in README.md + +terraform { + backend "gcs" { + bucket = var.bucket_name + prefix = var.bucket_prefix + credentials = var.bucket_credentials_json + } +} diff --git a/private-cluster-module/demo-namespace.tf b/private-cluster-module/demo-namespace.tf new file mode 100644 index 0000000..b8ff001 --- /dev/null +++ b/private-cluster-module/demo-namespace.tf @@ -0,0 +1,6 @@ +resource "kubernetes_namespace" "this-is-demo-cluster" { + metadata { + name = "this-is-demo-cluster" + } + depends_on = [module.gke] +} diff --git a/private-cluster-module/get-credentials.sh b/private-cluster-module/get-credentials.sh new file mode 100755 index 0000000..20f9494 --- /dev/null +++ b/private-cluster-module/get-credentials.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +if grep -H "opentofu.org" .terraform.lock.hcl > /dev/null; then + echo "trying to read tofu state" + gcloud container clusters get-credentials $(tofu output -raw kubernetes_cluster_name) --region $(tofu output -raw region) --project $(tofu output -raw project) +else + echo "trying to read terraform state" + gcloud container clusters get-credentials $(terraform output -raw kubernetes_cluster_name) --region $(terraform output -raw region) --project $(terraform output -raw project) +fi diff --git a/private-cluster-module/gke.tf b/private-cluster-module/gke.tf new file mode 100644 index 0000000..0ef98c4 --- /dev/null +++ b/private-cluster-module/gke.tf @@ -0,0 +1,73 @@ +locals { + cluster_name = "${var.environment_name}-k8s-${var.region}" +} + +module "gke" { + # https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/tree/master/modules/beta-private-cluster-update-variant + source = "terraform-google-modules/kubernetes-engine/google//modules/beta-private-cluster-update-variant" + version = "32.0.0" + project_id = var.project + network_project_id = var.host_project + name = local.cluster_name + region = var.region + zones = [var.zone] + network = google_compute_network.vpc.name + subnetwork = google_compute_subnetwork.subnet.name + ip_range_pods = "${var.project}-gke-pods" + ip_range_services = "${var.project}-gke-services" + http_load_balancing = true + horizontal_pod_autoscaling = true + enable_vertical_pod_autoscaling = true + network_policy = true + remove_default_node_pool = true + release_channel = "UNSPECIFIED" + create_service_account = false # means gsa, NOT k8s_sa + dns_cache = true + + deletion_protection = false # Use this only for testing purposess! + + maintenance_start_time = "2021-04-20T01:00:00Z" + maintenance_end_time = "2021-04-20T05:00:00Z" + maintenance_recurrence = "FREQ=WEEKLY;BYDAY=MO,TH,SU" + + master_authorized_networks = var.master_authorized_networks + + node_pools = [ + { + name = "node-pool-1" + machine_type = "n2-standard-8" + min_count = 1 + max_count = 2 + disk_size_gb = 30 + disk_type = "pd-standard" + image_type = "COS_CONTAINERD" + auto_repair = true + auto_upgrade = true + preemptible = true + initial_node_count = 1 + node_metadata = "GKE_METADATA" + }, + ] + + node_pools_oauth_scopes = { + all = [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/compute", + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring", + ] + } + + node_pools_tags = { + all = [ + "${local.cluster_name}-${var.region}", + "${local.cluster_name}-${var.region}-nodes" + ] + } + + depends_on = [ + google_compute_network.vpc, + google_compute_subnetwork.subnet + ] +} diff --git a/private-cluster-module/outputs.tf b/private-cluster-module/outputs.tf new file mode 100644 index 0000000..8bba745 --- /dev/null +++ b/private-cluster-module/outputs.tf @@ -0,0 +1,45 @@ +output "region" { + description = "GCloud Region" + value = var.region +} + +output "project" { + description = "GCloud Project ID" + value = var.project +} + +output "zone" { + description = "GCloud Project ID" + value = var.zone +} + +output "kubernetes_cluster_name" { + description = "GKE Cluster Name" + value = module.gke.name +} + +output "kubernetes_cluster_host" { + description = "GKE Cluster Host" + sensitive = true + value = module.gke.endpoint +} + +output "gcloud_gke_get_creds" { + description = "Command to get GKE credentials" + value = "gcloud container clusters get-credentials ${local.cluster_name} --region ${var.region} --project ${var.project}" +} + +output "gcloud_vpc_link" { + description = "VPC web ui link" + value = "https://console.cloud.google.com/networking/networks/list?project=${var.project}" +} + +output "gcloud_gke_link" { + description = "GKE web ui link" + value = "https://console.cloud.google.com/kubernetes/clusters/details/${var.region}/${local.cluster_name}/details?project=${var.project}" +} + +output "master_authorized_networks_config" { + description = "Here are networks that are allowed to reach your k8s API" + value = module.gke.master_authorized_networks_config +} diff --git a/private-cluster-module/providers.tf b/private-cluster-module/providers.tf new file mode 100644 index 0000000..b0d4d4c --- /dev/null +++ b/private-cluster-module/providers.tf @@ -0,0 +1,12 @@ +data "google_client_config" "default" {} + +provider "google" { + project = var.project + region = var.region +} + +provider "kubernetes" { + host = "https://${module.gke.endpoint}" + token = data.google_client_config.default.access_token + cluster_ca_certificate = base64decode(module.gke.ca_certificate) +} diff --git a/private-cluster-module/variables.tf b/private-cluster-module/variables.tf new file mode 100644 index 0000000..3f8b685 --- /dev/null +++ b/private-cluster-module/variables.tf @@ -0,0 +1,63 @@ +variable "project" { + type = string + description = "Google Project to create resources in" + default = "demo" +} + +variable "region" { + type = string + description = "The region to host the cluster in" + default = "europe-west1" +} + +variable "zone" { + type = string + description = "The region to host the cluster in" + default = "europe-west1-b" +} + +variable "environment_name" { + type = string + description = "Environment name" + default = "demo" +} + +variable "host_project" { + type = string + description = "The GCP project housing the VPC network to host the cluster in" + default = "demo" +} + +variable "vpc_network" { + type = string + description = "The GCP network to apply firewall rules in" + default = "demo-vpc" +} + +variable "master_authorized_networks" { + type = list(object({ cidr_block = string, display_name = string })) + description = "List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists)." + default = [ + { cidr_block = "1.1.1.1/32", display_name = "Use your IP/VPN IP here" }, + ] +} + +# Google bucket for terraform state + +variable "bucket_name" { + type = string + description = "Bucket name" + default = "your-project-bucket" +} + +variable "bucket_prefix" { + type = string + description = "Path to store your state in bucket" + default = "terraform/state/demo" +} + +variable "bucket_credentials_json" { + type = string + description = "Default path to your gcloud credentials json file" + default = "~/.config/gcloud/application_default_credentials.json" +} diff --git a/private-cluster-module/versions.tf b/private-cluster-module/versions.tf new file mode 100644 index 0000000..3bde509 --- /dev/null +++ b/private-cluster-module/versions.tf @@ -0,0 +1,21 @@ +terraform { + required_version = "~>1.8.0" + + required_providers { + # https://github.com/hashicorp/terraform-provider-google + google = { + source = "hashicorp/google" + version = "~>5.41.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = "~> 5.41" + } + + # https://github.com/hashicorp/terraform-provider-kubernetes + kubernetes = { + source = "hashicorp/kubernetes" + version = "~>2.32.0" + } + } +} diff --git a/private-cluster-module/vpc.tf b/private-cluster-module/vpc.tf new file mode 100644 index 0000000..af81ae0 --- /dev/null +++ b/private-cluster-module/vpc.tf @@ -0,0 +1,27 @@ +# https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_subnetwork +resource "google_project_service" "service_networking" { + service = "servicenetworking.googleapis.com" + project = var.project +} + +resource "google_compute_network" "vpc" { + name = var.vpc_network + auto_create_subnetworks = "false" +} + +resource "google_compute_subnetwork" "subnet" { + name = "${var.project}-subnet" + region = var.region + network = google_compute_network.vpc.name + ip_cidr_range = "10.1.0.0/24" + secondary_ip_range { + range_name = "${var.project}-gke-pods" + ip_cidr_range = "10.2.0.0/19" + } + secondary_ip_range { + range_name = "${var.project}-gke-services" + ip_cidr_range = "10.3.0.0/24" + } + private_ip_google_access = true + depends_on = [google_compute_network.vpc] +}