From cb468e04cad3c521a2c4f1a146eb51acca8b2b1d Mon Sep 17 00:00:00 2001 From: root Date: Wed, 21 Jun 2023 22:42:00 +0900 Subject: [PATCH] update --- README.md | 131 +++++++++++++++++- defaults/main.yml | 3 + tasks/dsk-common.yml | 11 +- tasks/dsk-docker-log-agent.yml | 9 +- tasks/dsk-docker-node-agent.yml | 4 +- tasks/dsk-docker-postgres-agent.yml | 19 ++- tasks/dsk-docker-trace-agent.yml | 7 +- tasks/dsk-log-agent.yml | 4 +- tasks/remove-datasaker.yml | 49 ++++++- templates/docker-log-agent-config.yml.j2 | 31 +++++ templates/docker-postgres-agent-config.yml.j2 | 2 +- templates/log-agent-config.yml.j2 | 42 +++--- templates/trace-agent-config.yml.j2 | 6 +- 13 files changed, 262 insertions(+), 56 deletions(-) create mode 100644 templates/docker-log-agent-config.yml.j2 diff --git a/README.md b/README.md index abc7a56..afdc8d5 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ The Ansible Datasaker role installs and configures the Datasaker Agent and integ - Requires Ansible v2.6+. - Supports most Debian Linux distributions. +- Supports Amazon Linux 2023 distributions. ### Installation @@ -19,6 +20,7 @@ ansible-galaxy install dsk_bot.datasaker To deploy the Datasaker Agent on hosts, add the Datasaker role and your API key to your playbook: +###### Host Agent Default Install Example ```yml - hosts: servers become: true @@ -26,16 +28,31 @@ To deploy the Datasaker Agent on hosts, add the Datasaker role and your API key - role: dsk_bot.datasaker vars: datasaker_api_key: "" - datasaker_agents: [""] + datasaker_agents: ["dsk-node-agent"] +``` +###### Docker Agent Default Install Example +```yml +- hosts: servers + become: true + roles: + - role: dsk_bot.datasaker + vars: + datasaker_api_key: "" + datasaker_docker_agents: ["dsk-docker-node-agent","dsk-docker-log-agent"] ``` -#### Role variables +#### Base Role variables -| Variable | Description | -|--------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Variable | Description | +|--------------------------------------------|--------------------------------------------------| |`datasaker_api_key`|Your Datasaker API key.| -|`datasaker_agents`|Set to Datasaker Agent.
`dsk-node-agent` `dsk-trace-agent` `dsk-log-agent` `dsk-postgres-agent` `dsk-plan-postgres-agent`
(Default) `dsk-node-agent`| -|`datagate_url`|The site of the Datasaker intake to send Agent data to.
(Default) `gate.kr.datasaker.io`| +|`datasaker_agents`|Set to Datasaker Host Agent.
`dsk-node-agent` `dsk-trace-agent` `dsk-log-agent` `dsk-postgres-agent` `dsk-plan-postgres-agent`
(Default) `dsk-node-agent`| +|`datasaker_docker_agents`|Set to Datasaker Docker Agent.
`dsk-docker-node-agent` `dsk-docker-trace-agent` `dsk-docker-log-agent` `dsk-docker-postgres-agent`
(Default) `dsk-docker-node-agent`| + + +#### Docker Agent Role variables +| Variable | Description | +|--------------------------------------------|--------------------------------------------------| +|`datasaker_docker_config_path`| Override the datasaker global config path.
(Default) `~/.datasaker`| +|`datasaker_docker_global_config`| Override the datasaker global config file name.
(Default) `~/.datasaker/config.yml`| +|`docker_default_path`| Override the docker containers path.
(Default) `/var/lib/docker/containers/`| +|`datasaker_docker_path`| Override the datasaker docker agent containers path.
(Default) `/var/datasaker`| +|`container_agent_restart_policy`| Override the restart policy for a `dsk-container-agent` container
(Default) `always`| +|`node_agent_restart_policy`| Override the restart policy for a `dsk-node-agent` container
(Default) `always`| +|`trace_agent_restart_policy`| Override the restart policy for a `dsk-trace-agent` container
(Default) `always`| +|`log_agent_restart_policy`| Override the restart policy for a `dsk-log-agent` container
(Default) `always`| +|`postgres_agent_restart_policy`| Override the restart policy for a `dsk-postgres-agent` container
(Default) `always`| +|`plan_postgres_agent_restart_policy`| Override the restart policy for a `dsk-plan-postgres-agent` container
(Default) `always`| +|`container_agent_log_level`| Override the `dsk-container-agent` log level
(Default) `INFO`| +|`node_agent_log_level`| Override the `dsk-node-agent` log level
(Default) `INFO`| +|`trace_agent_log_level`| Override the `dsk-trace-agent` log level
(Default) `INFO`| +|`log_agent_log_level`| Override the `dsk-log-agent` log level
(Default) `INFO`| +|`postgres_agent_log_level`| Override the `dsk-postgres-agent` log level
(Default) `INFO`| +|`plan_postgres_agent_log_level`| Override the `dsk-plan-postgres-agent` log level
(Default) `INFO`| + + + +#### Agents Setting Role variables +| Variable | Description | +|--------------------------------------------|--------------------------------------------------| +|`trace_sampling_rate`| Override The `dsk-trace-agent` sampling rate applied to the collector.
- When set to 100 or higher, all data is collected.
(Default) `10`| +|`log_collects`| An array of log collection configurations. Each item in the array includes the following. | +|`log_collects[*].paths`| An array of paths for log collection.
(Default) [`host-agent`]=`/var/log/*.log`, [`docker-agent`]=`/var/log/sample/*/*.log` | +|`log_collects[*].exclude_paths`| An array of paths to be excluded from the log collection. If the array is empty, no paths will be excluded.
(Default) `None` | +|`log_collects[*].keywords`| An array of keywords for filtering the logs. If the array is empty, no keyword filtering will be applied.
(Default) `None` | +|`log_collects[*].tag`| The tag for the log collection item.
(Default) `Default` | +|`log_collects[*].service.name`| The name of the service.
(Default) `default` | +|`log_collects[*].service.category`| The category of the service.
(Default) `etc` | +|`log_collects[*].service.type`| The type of the service.
(Default) `etc` | +|`log_collects[*].service.address`| The address of the service.
This field is optional and is used only for certain services that require an address. | +|`postgres_user_name`| Enter the Postgres user ID.
(Default) `None` | +|`postgres_user_password`| Enter the Postgres user password.
(Default) `None` | +|`postgres_database_address`| Enter the Postgres address.
(Default) `None` | +|`postgres_database_port`| Enter the Postgres port.
(Default) `None` | +|`plan_postgres_user_name`| Enter the Plan Postgres user ID.
(Default) `None` | +|`plan_postgres_user_password`| Enter the Plan Postgres user password.
(Default) `None` | +|`plan_postgres_database_address`| Enter the Plan Postgres address.
(Default) `None` | +|`plan_postgres_database_port`| Enter the Plan Postgres port.
(Default) `None` | +|`plan_postgres_database_name`| Enter the Plan Postgres database.
(Default) `None` | +|`plan_postgres_scrape_interval`| Override the Plan Postgres scrape interval
(Default) `30s` | +|`plan_postgres_scrape_timeout`| Override the Plan Postgres scrape timeout
(Default) `5s` | +|`plan_postgres_slow_query_standard`| Override the Plan Postgres slow query standard
(Default) `5s` | +|`plan_postgres_executor_number`| Override the Plan Postgres executor number
(Default) `10` | +|`plan_postgres_sender_number`| Override the Plan Postgres sender number
(Default) `10` | +|`plan_postgres_activity_query_buffer`| Override the Plan Postgres activity query buffer
(Default) `50` | +|`plan_postgres_plan_sender_buffer`| Override the Plan Postgres plan sender buffer
(Default) `50` | + +###### Agent Variables Example +```yml +- hosts: servers + become: true + roles: + - role: dsk_bot.datasaker + vars: + datasaker_api_key: "" + datasaker_docker_agents: ["dsk-docker-node-agent","dsk-docker-trace-agent","dsk-docker-log-agent","dsk-docker-postgres-agent"] + postgres_user_name: sample + postgres_user_password: 1q2w3e4r + postgres_database_address: 0.0.0.0 + postgres_database_port: 5432 + plan_postgres_user_name: sample + plan_postgres_user_password: 1q2w3e4r + plan_postgres_database_address: 0.0.0.0 + plan_postgres_database_name: sample + plan_postgres_database_port: 5432 + log_collects: + - paths: + - "/var/log/sample/*/*.log" + exclude_paths: [] + keywords: [] + tag: "Default" + service: + name: "default" + category: "etc" + type: "etc" + - paths: + - "/var/log/sample/b4d5ac015a5a*/*.log" + service: + name: "docker-test" + category: "database" + type: "postgres" + address: "0.0.0.0:5432" +``` ## Uninstallation @@ -69,9 +185,10 @@ In this example: uninstall: True datasaker_clean: True ``` - + diff --git a/defaults/main.yml b/defaults/main.yml index 98d28e2..5e91214 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -54,18 +54,21 @@ node_agent_image_tag: "latest" trace_agent_image_tag: "latest" log_agent_image_tag: "latest" postgres_agent_image_tag: "latest" +plan_postgres_agent_image_tag: "latest" container_agent_restart_policy: "always" node_agent_restart_policy: "always" trace_agent_restart_policy: "always" log_agent_restart_policy: "always" postgres_agent_restart_policy: "always" +plan_postgres_agent_restart_policy: "always" container_agent_log_level: "INFO" node_agent_log_level: "INFO" trace_agent_log_level: "INFO" log_agent_log_level: "INFO" postgres_agent_log_level: "INFO" +plan_postgres_agent_log_level: "INFO" uninstall: False datasaker_clean: False diff --git a/tasks/dsk-common.yml b/tasks/dsk-common.yml index 1cf4566..ed8a5b6 100644 --- a/tasks/dsk-common.yml +++ b/tasks/dsk-common.yml @@ -25,16 +25,21 @@ path: "{{ item }}" state: directory recurse: yes - owner: root - group: root with_items: - "{{ datasaker_docker_config_path }}" when: - datasaker_docker_agents +- name: "Touch agent config" + ansible.builtin.file: + path: "{{ datasaker_docker_global_config }}" + state: touch + when: + - datasaker_docker_agents + - name: "Setting global-config" template: src: global-config.yml.j2 dest: "{{ datasaker_docker_global_config }}" when: - - datasaker_docker_agents \ No newline at end of file + - datasaker_docker_agents diff --git a/tasks/dsk-docker-log-agent.yml b/tasks/dsk-docker-log-agent.yml index 874c5a9..578a538 100644 --- a/tasks/dsk-docker-log-agent.yml +++ b/tasks/dsk-docker-log-agent.yml @@ -9,6 +9,11 @@ with_items: - "{{ datasaker_docker_path }}/agent/dsk-log-agent" +- name: "Touch agent config" + ansible.builtin.file: + path: "{{ datasaker_docker_path }}/log-agent-config.yml" + state: touch + - name: "Setting dsk-log-agent config" template: src: log-agent-config.yml.j2 @@ -16,11 +21,11 @@ - name: Run Datasaker Log Agent docker_container: - name: dsk-log-agent + name: dsk-docker-log-agent image: "datasaker/dsk-log-agent:{{ trace_agent_image_tag }}" state: started detach: true - restart_policy: always + restart_policy: "{{ log_agent_restart_policy }}" env: DKS_LOG_LEVEL: "{{ log_agent_log_level }}" command: "-global.config=/etc/datasaker/global-config.yml -agent.config=/etc/datasaker/dsk-log-agent/agent-config.yml -mount.volume=true" diff --git a/tasks/dsk-docker-node-agent.yml b/tasks/dsk-docker-node-agent.yml index 64317b9..2d650a1 100644 --- a/tasks/dsk-docker-node-agent.yml +++ b/tasks/dsk-docker-node-agent.yml @@ -33,7 +33,7 @@ - name: "Run Datasaker Container Agent" ansible.builtin.docker_container: - name: dsk-container-agent + name: dsk-docker-container-agent image: "datasaker/dsk-container-agent:{{ container_agent_image_tag }}" state: started restart_policy: "{{ container_agent_restart_policy }}" @@ -52,7 +52,7 @@ - name: "Run Datasaker Node Agent" ansible.builtin.docker_container: - name: dsk-node-agent + name: dsk-docker-node-agent image: "datasaker/dsk-node-agent:{{ node_agent_image_tag }}" state: started restart_policy: "{{ node_agent_restart_policy }}" diff --git a/tasks/dsk-docker-postgres-agent.yml b/tasks/dsk-docker-postgres-agent.yml index 587c5c6..09e8b70 100644 --- a/tasks/dsk-docker-postgres-agent.yml +++ b/tasks/dsk-docker-postgres-agent.yml @@ -10,9 +10,14 @@ - "{{ datasaker_docker_path }}/agent/dsk-postgres-agent" - "{{ datasaker_docker_path }}/agent/dsk-plan-postgres-agent" +- name: "Touch agent config" + ansible.builtin.file: + path: "{{ datasaker_docker_path }}/dsk-postgres-config.yml" + state: touch + - name: "Setting dsk-postgres-agent config" template: - src: postgres-agent-config.yml.j2 + src: docker-postgres-agent-config.yml.j2 dest: "{{ datasaker_docker_path }}/dsk-postgres-config.yml" - name: "Setting plan-dsk-postgres-agent config" @@ -22,10 +27,10 @@ - name: "Run dsk-postgres-agent container" docker_container: - name: dsk-postgres-agent - image: datasaker/dsk-postgres-agent:rel0.1.2 + name: dsk-docker-postgres-agent + image: "datasaker/dsk-postgres-agent:{{ postgres_agent_image_tag }}" state: started - restart_policy: always + restart_policy: "{{ postgres_agent_restart_policy }}" detach: true env: DKS_LOG_LEVEL: "info" @@ -39,10 +44,10 @@ - name: "Run dsk-plan-postgres-agent container" docker_container: - name: dsk-plan-postgres-agent - image: datasaker/dsk-plan-postgres-agent:latest + name: dsk-docker-plan-postgres-agent + image: "datasaker/dsk-plan-postgres-agent:{{ plan_postgres_agent_image_tag }}" state: started - restart_policy: always + restart_policy: "{{ plan_postgres_agent_restart_policy }}" detach: true env: DKS_LOG_LEVEL: "info" diff --git a/tasks/dsk-docker-trace-agent.yml b/tasks/dsk-docker-trace-agent.yml index d97a059..8896f0b 100644 --- a/tasks/dsk-docker-trace-agent.yml +++ b/tasks/dsk-docker-trace-agent.yml @@ -9,6 +9,11 @@ with_items: - "{{ datasaker_docker_path }}/agent/dsk-trace-agent" +- name: "Touch agent config" + ansible.builtin.file: + path: "{{ datasaker_docker_path }}/trace-agent-config.yml" + state: touch + - name: "Setting dsk-trace-agent config" template: src: trace-agent-config.yml.j2 @@ -16,7 +21,7 @@ - name: "Run Datasaker trace Agent" ansible.builtin.docker_container: - name: dsk-trace-agent + name: dsk-docker-trace-agent image: "datasaker/dsk-trace-agent:{{ trace_agent_image_tag }}" state: started restart_policy: "{{ trace_agent_restart_policy }}" diff --git a/tasks/dsk-log-agent.yml b/tasks/dsk-log-agent.yml index 5dfff23..b2532fc 100644 --- a/tasks/dsk-log-agent.yml +++ b/tasks/dsk-log-agent.yml @@ -4,12 +4,10 @@ path: "/etc/datasaker/{{ item }}" state: directory recurse: yes - owner: root - group: root with_items: - "dsk-log-agent" - name: "Setting dsk-log-agent config" template: src: log-agent-config.yml.j2 - dest: "/etc/datasaker/dsk-log-agent/agent-config.yml" \ No newline at end of file + dest: "/etc/datasaker/dsk-log-agent/agent-config.yml" diff --git a/tasks/remove-datasaker.yml b/tasks/remove-datasaker.yml index cb489e5..5b52b35 100644 --- a/tasks/remove-datasaker.yml +++ b/tasks/remove-datasaker.yml @@ -5,6 +5,8 @@ state: absent with_items: - "{{ datasaker_agents }}" + when: + - datasaker_agents - name: Remove Datasaker Agent Directory ansible.builtin.file: @@ -12,6 +14,8 @@ state: absent with_items: - "{{ datasaker_agents }}" + when: + - datasaker_agents - name: Stop agent service service: @@ -20,6 +24,8 @@ state: stopped with_items: - "{{ datasaker_agents }}" + when: + - datasaker_agents ignore_errors: true - name: "Uninstall datasaker agent" @@ -29,6 +35,27 @@ update_cache: yes with_items: - "{{ datasaker_agents }}" + when: + - datasaker_agents + ignore_errors: true + +- name: Stop the Docker container + docker_container: + name: "{{ item }}" + state: absent + with_items: + - "dsk-docker-container-agent" + - "{{ datasaker_docker_agents }}" + when: + - datasaker_docker_agents + ignore_errors: true + +- name: Stop the Docker plan-postgres-agent container + docker_container: + name: "dsk-docker-plan-postgres-agent" + state: absent + when: + - '"dsk-docker-postgres-agent" in datasaker_docker_agents' ignore_errors: true - name: Remove Datasaker Agent Directory @@ -37,6 +64,8 @@ state: absent with_items: - "{{ datasaker_agents }}" + when: + - datasaker_agents - name: Remove Datasaker All Directory ansible.builtin.file: @@ -46,32 +75,40 @@ - /etc/datasaker - /var/log/datasaker - /var/datasaker - - ~/.datasaker - "{{ apt_sources_list }}" - "{{ apt_trusted_d_keyring }}" - "{{ apt_usr_share_keyring }}" - when: datasaker_clean == True + when: + - datasaker_clean == True + - datasaker_agents - name: "Remove datasaker_docker_user" ansible.builtin.user: name: "{{datasaker_docker_user}}" state: absent - when: datasaker_clean == True + when: + - datasaker_clean == True + - datasaker_docker_agents ignore_errors: yes - name: "Remove datasaker_docker_group" ansible.builtin.group: name: "{{datasaker_docker_group}}" state: absent - when: datasaker_clean == True + when: + - datasaker_clean == True + - datasaker_docker_agents ignore_errors: yes - name: "Remove datasaker_docker_directory" ansible.builtin.file: name: "{{ item }}" state: absent - when: datasaker_clean == True - ignore_errors: yes with_items: - "{{ datasaker_docker_config_path }}" - "{{ datasaker_docker_path }}" + when: + - datasaker_clean == True + - datasaker_docker_agents + ignore_errors: yes + diff --git a/templates/docker-log-agent-config.yml.j2 b/templates/docker-log-agent-config.yml.j2 new file mode 100644 index 0000000..8789c09 --- /dev/null +++ b/templates/docker-log-agent-config.yml.j2 @@ -0,0 +1,31 @@ +agent: + metadata: + agent_name: "{{ log_agent_name | default('dsk-log-agent') }}" + collect: +{% for collect_item in log_collects %} + - paths: +{% for path in collect_item.paths %} + - "{{ path | default('/var/log/sample/*/*.log') }}" +{% endfor %} +{% if collect_item.exclude_paths is defined %} + exclude_paths: +{% for exclude_path in collect_item.exclude_paths %} + - "{{ exclude_path | default('[]') }}" +{% endfor %} +{% endif %} +{% if collect_item.keywords is defined %} + keywords: +{% for keyword in collect_item.keywords %} + - "{{ keyword | default('[]') }}" +{% endfor %} +{% endif %} + tag: "{{ collect_item.tag | default('Default') }}" + service: + name: "{{ collect_item.service.name | default('default') }}" + category: "{{ collect_item.service.category | default('etc') }}" + type: "{{ collect_item.service.type | default('etc') }}" +{% if collect_item.service.address is defined %} + address: "{{ collect_item.service.address }}" +{% endif %} +{% endfor %} + diff --git a/templates/docker-postgres-agent-config.yml.j2 b/templates/docker-postgres-agent-config.yml.j2 index b5563c0..7e4e166 100644 --- a/templates/docker-postgres-agent-config.yml.j2 +++ b/templates/docker-postgres-agent-config.yml.j2 @@ -1,6 +1,6 @@ agent: metadata: - agent_name: "{{ docker_postgres_agent_name | default('dsk-postgres-agent') }}" + agent_name: "{{ docker_postgres_agent_name | default('dsk-docker-postgres-agent') }}" option: exporter_config: command: "/etc/datasaker/target-exporter" diff --git a/templates/log-agent-config.yml.j2 b/templates/log-agent-config.yml.j2 index 951b196..c9d2a1f 100644 --- a/templates/log-agent-config.yml.j2 +++ b/templates/log-agent-config.yml.j2 @@ -2,32 +2,30 @@ agent: metadata: agent_name: "{{ log_agent_name | default('dsk-log-agent') }}" collect: +{% for collect_item in log_collects %} - paths: -{% if paths is defined and paths | length > 0 %} -{% for path in paths %} - - "{{ path | default('') }}" +{% for path in collect_item.paths %} + - "{{ path | default('/var/log/*.log') }}" {% endfor %} -{% else %} - - /var/log/sample/*/*.log -{% endif %} -{% if exclude_paths is defined and exclude_paths | length > 0 %} +{% if collect_item.exclude_paths is defined %} exclude_paths: -{% for exclude_path in exclude_paths %} - - "{{ exclude_path | default('') }}" +{% for exclude_path in collect_item.exclude_paths %} + - "{{ exclude_path | default('[]') }}" {% endfor %} -{% else %} - exclude_paths: [] {% endif %} -{% if keywords is defined and keywords | length > 0 %} - keywords: -{% for keyword in keywords %} - - "{{ keyword | default('ERROR') }}" +{% if collect_item.keywords is defined %} + keywords: +{% for keyword in collect_item.keywords %} + - "{{ keyword | default('[]') }}" {% endfor %} -{% else %} - keywords: [] -{% endif %} - tag: "{{ log_agent_tag | default('Default') }}" +{% endif %} + tag: "{{ collect_item.tag | default('Default') }}" service: - name: "{{ log_agent_service_name | default('default') }}" - category: "{{ log_agent_service_category | default('etc') }}" - type: "{{ log_agent_service_type | default('etc') }}" + name: "{{ collect_item.service.name | default('default') }}" + category: "{{ collect_item.service.category | default('etc') }}" + type: "{{ collect_item.service.type | default('etc') }}" +{% if collect_item.service.address is defined %} + address: "{{ collect_item.service.address }}" +{% endif %} +{% endfor %} + diff --git a/templates/trace-agent-config.yml.j2 b/templates/trace-agent-config.yml.j2 index 5b92245..2c41a7d 100644 --- a/templates/trace-agent-config.yml.j2 +++ b/templates/trace-agent-config.yml.j2 @@ -1,5 +1,7 @@ agent: metadata: + agent_name: "{{ trace_agent_name | default('trace-agent') }}" + cluster_id: "{{ trace_agent_cluster_id | default('unknown_cluster') }}" option: - agent_name: "{{ trace_agent_name | default('trace-agent') }}" - cluster_id: "{{ trace_agent_cluster_id | default('unknown_cluster') }}" \ No newline at end of file + collector_config: + sampling_rate: "{{ trace_sampling_rate | default('10') }}"