add bucket and private-cluster-module

This commit is contained in:
Aleksei Krugliak 2024-09-01 15:19:36 +04:00
parent a77c2a72f2
commit 4f58bc6b17
16 changed files with 573 additions and 0 deletions

81
bucket/README.md Normal file
View File

@ -0,0 +1,81 @@
# bucket
You should be an admin/owner of your sandbox or have enougth permissions to create a bucket.
If you use a lot of accounts and specific IAM rules, you can check the permissions of the user with the following command:
```shell
gcloud projects get-iam-policy <GCP_PROJECT> --format=json | jq '.bindings[] | select(.members[] | contains("user:<YOUR@gmail.com>"))'
```
`"role": "roles/owner"` would be great for you.
## How to use the code
1. Create `terraform.tfvars` file with a few variables
```shell
project = "your-gcp-project"
iam_user_email = "your@gmail.com"
```
`region` is optional.
2. Create bucket
All commands will be applied via Terraform 1.8.0 or via OpenTofu, the same version.
I use alias `t` for the commands.
```shell
t init
t apply
```
2. Destrouy bucket sfter your tests
```shell
t destroy
```
<!-- BEGIN_TF_DOCS -->
## Requirements
| Name | Version |
|------|---------|
| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | ~>1.8.0 |
| <a name="requirement_google"></a> [google](#requirement\_google) | ~>5.41.0 |
## Providers
| Name | Version |
|------|---------|
| <a name="provider_google"></a> [google](#provider\_google) | 5.41.0 |
## Modules
| Name | Source | Version |
|------|--------|---------|
| <a name="module_bucket"></a> [bucket](#module\_bucket) | terraform-google-modules/cloud-storage/google//modules/simple_bucket | ~> 6.0 |
## Resources
| Name | Type |
|------|------|
| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source |
## Inputs
| Name | Description | Type | Default | Required |
|------|-------------|------|---------|:--------:|
| <a name="input_iam_user_email"></a> [iam\_user\_email](#input\_iam\_user\_email) | Your gcloud account | `string` | `"demo@gmail.com"` | no |
| <a name="input_project"></a> [project](#input\_project) | Google Project to create resources in | `string` | `"demo"` | no |
| <a name="input_region"></a> [region](#input\_region) | The region to host the cluster in | `string` | `"europe-west1"` | no |
## Outputs
| Name | Description |
|------|-------------|
| <a name="output_gcloud_bucket_link"></a> [gcloud\_bucket\_link](#output\_gcloud\_bucket\_link) | Bucket web ui link |
| <a name="output_project"></a> [project](#output\_project) | GCloud Project ID |
| <a name="output_region"></a> [region](#output\_region) | GCloud Region |
<!-- END_TF_DOCS -->

18
bucket/main.tf Normal file
View File

@ -0,0 +1,18 @@
module "bucket" {
# https://registry.terraform.io/modules/terraform-google-modules/cloud-storage/google/6.0.1/submodules/simple_bucket?tab=inputs
source = "terraform-google-modules/cloud-storage/google//modules/simple_bucket"
version = "~> 6.0"
name = "${var.project}-bucket"
project_id = var.project
location = "eu"
force_destroy = true # Use this only for testing purposes
iam_members = [{
role = "roles/storage.objectAdmin"
member = "user:${var.iam_user_email}"
}]
autoclass = true
}

14
bucket/outputs.tf Normal file
View File

@ -0,0 +1,14 @@
output "region" {
description = "GCloud Region"
value = var.region
}
output "project" {
description = "GCloud Project ID"
value = var.project
}
output "gcloud_bucket_link" {
description = "Bucket web ui link"
value = "https://console.cloud.google.com/storage/browser/${module.bucket.name};tab=objects?forceOnBucketsSortingFiltering=true&project=${var.project}&prefix=&forceOnObjectsSortingFiltering=false"
}

6
bucket/providers.tf Normal file
View File

@ -0,0 +1,6 @@
data "google_client_config" "default" {}
provider "google" {
project = var.project
region = var.region
}

17
bucket/variables.tf Normal file
View File

@ -0,0 +1,17 @@
variable "project" {
type = string
description = "Google Project to create resources in"
default = "demo"
}
variable "region" {
type = string
description = "The region to host the cluster in"
default = "europe-west1"
}
variable "iam_user_email" {
type = string
description = "Your gcloud account"
default = "demo@gmail.com"
}

11
bucket/versions.tf Normal file
View File

@ -0,0 +1,11 @@
terraform {
required_version = "~>1.8.0"
required_providers {
# https://github.com/hashicorp/terraform-provider-google
google = {
source = "hashicorp/google"
version = "~>5.41.0"
}
}
}

View File

@ -0,0 +1,160 @@
# private-cluster-module
## What is created ty thah template
This example is using remote bucket state. You can modify this manually or not.
1. VPC network with one subnet
2. GKE cluster with external endpoint and only authorized networks access.
3. Workload nodepool with one node
3. Namespace `this-is-demo-cluster`
## How to use the code
### Optional bucket usage.
If you would try to use bucket state, you need to uncomment code block in [backend.tf](backend.tf).
Then go to [../bucket](../bucket/) directory and create the bucket by following the instructions at [README](../bucket/README.md).
Then return to this directory.
1. Create `terraform.tfvars` file with a few variables
```shell
project = "your-gcp-project"
host_project = "your-gcp-project"
master_authorized_networks = [
{ cidr_block = "0.0.0.1/32", display_name = "your current IP" }
]
```
`region`, `zone` and `environment_name` are optional
Your current IP you can get via this command for example:
```shell
curl -s ifconfig.me
```
2. Create cluster
All commands will be applied via Terraform 1.8.0 or via OpenTofu, the same version.
I use alias `t` for the commands.
```shell
t init
t apply
```
3. Get the credentials for the new cluster (configure kubeconfig)
You can see all useful commands and links in the output:
```shell
t output
```
There is a manual command:
```shell
gcloud container clusters get-credentials $(t output -raw kubernetes_cluster_name) --region $(t output -raw zone) --project $(t output -raw project)
```
Or just use `./get-credentials.sh`
4. Destroy all resources
```shell
t destroy
```
## Additional info
Some manual tests.
### terraform
✅ create cluster<br>
`./get-credentials.sh`<br>
✅ try access from different networks<br>
✅ manual cred command<br>
✅ output `gcloud_gke_get_creds` command<br>
✅ output links<br>
✅ create simple nginx pod `kubectl run nginx --image=nginx:latest -n default && kubectl get po -w`<br>
`flux install && kubectl get po -n flux-system -w`<br>
✅ destroy cluster<br>
✅ test bucket backend
### opentofu
✅ create cluster<br>
`./get-credentials.sh`<br>
✅ try access from different networks<br>
✅ manual cred command<br>
✅ output `gcloud_gke_get_creds` command<br>
✅ output links<br>
✅ create simple nginx pod `kubectl run nginx --image=nginx:latest -n default && kubectl get po -w`<br>
`flux install && kubectl get po -n flux-system -w`<br>
✅ destroy cluster<br>
✅ test bucket backend
<!-- BEGIN_TF_DOCS -->
## Requirements
| Name | Version |
|------|---------|
| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | ~>1.8.0 |
| <a name="requirement_google"></a> [google](#requirement\_google) | ~>5.41.0 |
| <a name="requirement_google-beta"></a> [google-beta](#requirement\_google-beta) | ~> 5.41 |
| <a name="requirement_kubernetes"></a> [kubernetes](#requirement\_kubernetes) | ~>2.32.0 |
## Providers
| Name | Version |
|------|---------|
| <a name="provider_google"></a> [google](#provider\_google) | 5.41.0 |
| <a name="provider_kubernetes"></a> [kubernetes](#provider\_kubernetes) | 2.32.0 |
## Modules
| Name | Source | Version |
|------|--------|---------|
| <a name="module_gke"></a> [gke](#module\_gke) | terraform-google-modules/kubernetes-engine/google//modules/beta-private-cluster-update-variant | 32.0.0 |
## Resources
| Name | Type |
|------|------|
| [google_compute_network.vpc](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_network) | resource |
| [google_compute_subnetwork.subnet](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_subnetwork) | resource |
| [google_project_service.service_networking](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_service) | resource |
| [kubernetes_namespace.this-is-demo-cluster](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace) | resource |
| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source |
## Inputs
| Name | Description | Type | Default | Required |
|------|-------------|------|---------|:--------:|
| <a name="input_bucket_credentials_json"></a> [bucket\_credentials\_json](#input\_bucket\_credentials\_json) | Default path to your gcloud credentials json file | `string` | `"~/.config/gcloud/application_default_credentials.json"` | no |
| <a name="input_bucket_name"></a> [bucket\_name](#input\_bucket\_name) | Bucket name | `string` | `"your-project-bucket"` | no |
| <a name="input_bucket_prefix"></a> [bucket\_prefix](#input\_bucket\_prefix) | Path to store your state in bucket | `string` | `"terraform/state/demo"` | no |
| <a name="input_environment_name"></a> [environment\_name](#input\_environment\_name) | Environment name | `string` | `"demo"` | no |
| <a name="input_host_project"></a> [host\_project](#input\_host\_project) | The GCP project housing the VPC network to host the cluster in | `string` | `"demo"` | no |
| <a name="input_master_authorized_networks"></a> [master\_authorized\_networks](#input\_master\_authorized\_networks) | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | <pre>[<br> {<br> "cidr_block": "1.1.1.1/32",<br> "display_name": "Use your IP/VPN IP here"<br> }<br>]</pre> | no |
| <a name="input_project"></a> [project](#input\_project) | Google Project to create resources in | `string` | `"demo"` | no |
| <a name="input_region"></a> [region](#input\_region) | The region to host the cluster in | `string` | `"europe-west1"` | no |
| <a name="input_vpc_network"></a> [vpc\_network](#input\_vpc\_network) | The GCP network to apply firewall rules in | `string` | `"demo-vpc"` | no |
| <a name="input_zone"></a> [zone](#input\_zone) | The region to host the cluster in | `string` | `"europe-west1-b"` | no |
## Outputs
| Name | Description |
|------|-------------|
| <a name="output_gcloud_gke_get_creds"></a> [gcloud\_gke\_get\_creds](#output\_gcloud\_gke\_get\_creds) | Command to get GKE credentials |
| <a name="output_gcloud_gke_link"></a> [gcloud\_gke\_link](#output\_gcloud\_gke\_link) | GKE web ui link |
| <a name="output_gcloud_vpc_link"></a> [gcloud\_vpc\_link](#output\_gcloud\_vpc\_link) | VPC web ui link |
| <a name="output_kubernetes_cluster_host"></a> [kubernetes\_cluster\_host](#output\_kubernetes\_cluster\_host) | GKE Cluster Host |
| <a name="output_kubernetes_cluster_name"></a> [kubernetes\_cluster\_name](#output\_kubernetes\_cluster\_name) | GKE Cluster Name |
| <a name="output_master_authorized_networks_config"></a> [master\_authorized\_networks\_config](#output\_master\_authorized\_networks\_config) | Here are networks that are allowed to reach your k8s API |
| <a name="output_project"></a> [project](#output\_project) | GCloud Project ID |
| <a name="output_region"></a> [region](#output\_region) | GCloud Region |
| <a name="output_zone"></a> [zone](#output\_zone) | GCloud Project ID |
<!-- END_TF_DOCS -->

View File

@ -0,0 +1,10 @@
# Optionallly you can use bucket as backend
# If you would to do that - uncomment code below and follow instructions in README.md
terraform {
backend "gcs" {
bucket = var.bucket_name
prefix = var.bucket_prefix
credentials = var.bucket_credentials_json
}
}

View File

@ -0,0 +1,6 @@
resource "kubernetes_namespace" "this-is-demo-cluster" {
metadata {
name = "this-is-demo-cluster"
}
depends_on = [module.gke]
}

View File

@ -0,0 +1,9 @@
#!/bin/bash
if grep -H "opentofu.org" .terraform.lock.hcl > /dev/null; then
echo "trying to read tofu state"
gcloud container clusters get-credentials $(tofu output -raw kubernetes_cluster_name) --region $(tofu output -raw region) --project $(tofu output -raw project)
else
echo "trying to read terraform state"
gcloud container clusters get-credentials $(terraform output -raw kubernetes_cluster_name) --region $(terraform output -raw region) --project $(terraform output -raw project)
fi

View File

@ -0,0 +1,73 @@
locals {
cluster_name = "${var.environment_name}-k8s-${var.region}"
}
module "gke" {
# https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/tree/master/modules/beta-private-cluster-update-variant
source = "terraform-google-modules/kubernetes-engine/google//modules/beta-private-cluster-update-variant"
version = "32.0.0"
project_id = var.project
network_project_id = var.host_project
name = local.cluster_name
region = var.region
zones = [var.zone]
network = google_compute_network.vpc.name
subnetwork = google_compute_subnetwork.subnet.name
ip_range_pods = "${var.project}-gke-pods"
ip_range_services = "${var.project}-gke-services"
http_load_balancing = true
horizontal_pod_autoscaling = true
enable_vertical_pod_autoscaling = true
network_policy = true
remove_default_node_pool = true
release_channel = "UNSPECIFIED"
create_service_account = false # means gsa, NOT k8s_sa
dns_cache = true
deletion_protection = false # Use this only for testing purposess!
maintenance_start_time = "2021-04-20T01:00:00Z"
maintenance_end_time = "2021-04-20T05:00:00Z"
maintenance_recurrence = "FREQ=WEEKLY;BYDAY=MO,TH,SU"
master_authorized_networks = var.master_authorized_networks
node_pools = [
{
name = "node-pool-1"
machine_type = "n2-standard-8"
min_count = 1
max_count = 2
disk_size_gb = 30
disk_type = "pd-standard"
image_type = "COS_CONTAINERD"
auto_repair = true
auto_upgrade = true
preemptible = true
initial_node_count = 1
node_metadata = "GKE_METADATA"
},
]
node_pools_oauth_scopes = {
all = [
"https://www.googleapis.com/auth/cloud-platform",
"https://www.googleapis.com/auth/compute",
"https://www.googleapis.com/auth/devstorage.read_only",
"https://www.googleapis.com/auth/logging.write",
"https://www.googleapis.com/auth/monitoring",
]
}
node_pools_tags = {
all = [
"${local.cluster_name}-${var.region}",
"${local.cluster_name}-${var.region}-nodes"
]
}
depends_on = [
google_compute_network.vpc,
google_compute_subnetwork.subnet
]
}

View File

@ -0,0 +1,45 @@
output "region" {
description = "GCloud Region"
value = var.region
}
output "project" {
description = "GCloud Project ID"
value = var.project
}
output "zone" {
description = "GCloud Project ID"
value = var.zone
}
output "kubernetes_cluster_name" {
description = "GKE Cluster Name"
value = module.gke.name
}
output "kubernetes_cluster_host" {
description = "GKE Cluster Host"
sensitive = true
value = module.gke.endpoint
}
output "gcloud_gke_get_creds" {
description = "Command to get GKE credentials"
value = "gcloud container clusters get-credentials ${local.cluster_name} --region ${var.region} --project ${var.project}"
}
output "gcloud_vpc_link" {
description = "VPC web ui link"
value = "https://console.cloud.google.com/networking/networks/list?project=${var.project}"
}
output "gcloud_gke_link" {
description = "GKE web ui link"
value = "https://console.cloud.google.com/kubernetes/clusters/details/${var.region}/${local.cluster_name}/details?project=${var.project}"
}
output "master_authorized_networks_config" {
description = "Here are networks that are allowed to reach your k8s API"
value = module.gke.master_authorized_networks_config
}

View File

@ -0,0 +1,12 @@
data "google_client_config" "default" {}
provider "google" {
project = var.project
region = var.region
}
provider "kubernetes" {
host = "https://${module.gke.endpoint}"
token = data.google_client_config.default.access_token
cluster_ca_certificate = base64decode(module.gke.ca_certificate)
}

View File

@ -0,0 +1,63 @@
variable "project" {
type = string
description = "Google Project to create resources in"
default = "demo"
}
variable "region" {
type = string
description = "The region to host the cluster in"
default = "europe-west1"
}
variable "zone" {
type = string
description = "The region to host the cluster in"
default = "europe-west1-b"
}
variable "environment_name" {
type = string
description = "Environment name"
default = "demo"
}
variable "host_project" {
type = string
description = "The GCP project housing the VPC network to host the cluster in"
default = "demo"
}
variable "vpc_network" {
type = string
description = "The GCP network to apply firewall rules in"
default = "demo-vpc"
}
variable "master_authorized_networks" {
type = list(object({ cidr_block = string, display_name = string }))
description = "List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists)."
default = [
{ cidr_block = "1.1.1.1/32", display_name = "Use your IP/VPN IP here" },
]
}
# Google bucket for terraform state
variable "bucket_name" {
type = string
description = "Bucket name"
default = "your-project-bucket"
}
variable "bucket_prefix" {
type = string
description = "Path to store your state in bucket"
default = "terraform/state/demo"
}
variable "bucket_credentials_json" {
type = string
description = "Default path to your gcloud credentials json file"
default = "~/.config/gcloud/application_default_credentials.json"
}

View File

@ -0,0 +1,21 @@
terraform {
required_version = "~>1.8.0"
required_providers {
# https://github.com/hashicorp/terraform-provider-google
google = {
source = "hashicorp/google"
version = "~>5.41.0"
}
google-beta = {
source = "hashicorp/google-beta"
version = "~> 5.41"
}
# https://github.com/hashicorp/terraform-provider-kubernetes
kubernetes = {
source = "hashicorp/kubernetes"
version = "~>2.32.0"
}
}
}

View File

@ -0,0 +1,27 @@
# https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_subnetwork
resource "google_project_service" "service_networking" {
service = "servicenetworking.googleapis.com"
project = var.project
}
resource "google_compute_network" "vpc" {
name = var.vpc_network
auto_create_subnetworks = "false"
}
resource "google_compute_subnetwork" "subnet" {
name = "${var.project}-subnet"
region = var.region
network = google_compute_network.vpc.name
ip_cidr_range = "10.1.0.0/24"
secondary_ip_range {
range_name = "${var.project}-gke-pods"
ip_cidr_range = "10.2.0.0/19"
}
secondary_ip_range {
range_name = "${var.project}-gke-services"
ip_cidr_range = "10.3.0.0/24"
}
private_ip_google_access = true
depends_on = [google_compute_network.vpc]
}