pkg/controller/podautoscaler/horizontal.go:169

// Run begins watching and syncing.
func (a *HorizontalController) Run(stopCh <-chan struct{}) {
	defer utilruntime.HandleCrash()
	defer a.queue.ShutDown()

	glog.Infof("Starting HPA controller")
	defer glog.Infof("Shutting down HPA controller")

	if !controller.WaitForCacheSync("HPA", stopCh, a.hpaListerSynced) {
		return
	}

	// start a single worker (we may wish to start more in the future)
	go wait.Until(a.worker, time.Second, stopCh)

	<-stopCh
}

utilization raw

For per-pod resource metrics (like CPU), the controller fetches the metrics from the resource metrics API for each pod targeted by the HorizontalPodAutoscaler. Then, if a target utilization value is set, the controller calculates the utilization value as a percentage of the equivalent resource request on the containers in each pod. If a target raw value is set, the raw metric values are used directly. The controller then takes the mean of the utilization or the raw value (depending on the type of target specified) across all targeted pods, and produces a ratio used to scale the number of desired replicas.

do not have the relevant resource request set, ==> no action

–horizontal-pod-autoscaler-use-rest-clients WARNING: alpha feature. If set to true, causes the horizontal pod autoscaler controller to use REST clients through the kube-aggregator, instead of using the legacy metrics client through the API server proxy. This is required for custom metrics support in the horizontal pod autoscaler. (default true)

GET /apis/autoscaling/v1/namespaces/{namespace}/horizontalpodautoscalers

LIST curl -H “Authorization: Bearer $token” -k https://apigw.example.com:8143/clusters/system/apis/autoscaling/v1/namespaces/dex/horizontalpodautoscalers

READ curl -H “Authorization: Bearer $token” -k https://apigw.example.com:8143/clusters/system/apis/autoscaling/v1/namespaces/dex/horizontalpodautoscalers/php-apache

POST curl -X POST -H “Content-Type: application/json” -H “Accept: application/json” -H “Authorization: Bearer $token” -k https://apigw.example.com:8143/clusters/system/apis/autoscaling/v1/namespaces/dex/horizontalpodautoscalers –data ‘{“apiVersion”:”autoscaling/v1”,”kind”:”HorizontalPodAutoscaler”,”metadata”:{“name”:”php-apache”,”namespace”:”dex”},”spec”:{“maxReplicas”:10,”minReplicas”:1,”scaleTargetRef”:{“apiVersion”:”extensions/v1beta1”,”kind”:”Deployment”,”name”:”php-apache”},”targetCPUUtilizationPercentage”:50}}’

对应yaml格式

apiVersion: autoscaling/v1 kind: HorizontalPodAutoscaler metadata: name: php-apache namespace: dex spec: maxReplicas: 10 #最大副本数 minReplicas: 1 #最小副本数 scaleTargetRef: apiVersion: extensions/v1beta1 kind: Deployment name: php-apache #对应的deployment targetCPUUtilizationPercentage: 50 # cpu 的%比

PATCH curl -k -X PATCH -H “Accept: application/json” -H “Accept: application/json” -H “Authorization: Bearer $token” -H “Content-Type: application/strategic-merge-patch+json” https://apigw.example.com:8143/clusters/system/apis/autoscaling/v1/namespaces/dex/horizontalpodautoscalers/php-apache –data ‘{“spec”:{“targetCPUUtilizationPercentage”:60}}’

DELETE curl -k -X DELETE -H “Accept: application/json” -H “Accept: application/json” -H “Authorization: Bearer $token” https://apigw.example.com:8143/clusters/system//apis/autoscaling/v1/namespaces/dex/horizontalpodautoscalers/php-apache

更详细的api说明请见 https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#horizontalpodautoscaler-v1-autoscaling

[bottle@start /etc/docker]$ kubectl get hpa -w NAME REFERENCE TARGETS MINPODS MAXPODS REPLICAS AGE php-apache Deployment/php-apache 0%/50% 1 10 1 1h php-apache Deployment/php-apache 228%/50% 1 10 1 1h php-apache Deployment/php-apache 419%/50% 1 10 4 1h php-apache Deployment/php-apache 419%/50% 1 10 4 1h php-apache Deployment/php-apache 117%/50% 1 10 4 1h php-apache Deployment/php-apache 117%/50% 1 10 4 1h php-apache Deployment/php-apache 122%/50% 1 10 4 1h php-apache Deployment/php-apache 122%/50% 1 10 4 1h php-apache Deployment/php-apache 123%/50% 1 10 4 1h php-apache Deployment/php-apache 123%/50% 1 10 8 1h php-apache Deployment/php-apache 77%/50% 1 10 8 1h php-apache Deployment/php-apache 77%/50% 1 10 8 1h php-apache Deployment/php-apache 61%/50% 1 10 8 1h php-apache Deployment/php-apache 61%/50% 1 10 8 1h php-apache Deployment/php-apache 61%/50% 1 10 8 1h php-apache Deployment/php-apache 61%/50% 1 10 8 1h php-apache Deployment/php-apache 61%/50% 1 10 8 1h php-apache Deployment/php-apache 61%/50% 1 10 10 1h php-apache Deployment/php-apache 52%/50% 1 10 10 1h php-apache Deployment/php-apache 52%/50% 1 10 10 1h php-apache Deployment/php-apache 49%/50% 1 10 10 1h php-apache Deployment/php-apache 49%/50% 1 10 10 1h php-apache Deployment/php-apache 49%/50% 1 10 10 1h php-apache Deployment/php-apache 49%/50% 1 10 10 1h php-apache Deployment/php-apache 48%/50% 1 10 10 1h php-apache Deployment/php-apache 48%/50% 1 10 10 1h php-apache Deployment/php-apache 48%/50% 1 10 10 1h php-apache Deployment/php-apache 48%/50% 1 10 10 1h php-apache Deployment/php-apache 49%/50% 1 10 10 1h php-apache Deployment/php-apache 48%/50% 1 10 10 1h php-apache Deployment/php-apache 49%/50% 1 10 10 1h php-apache Deployment/php-apache 48%/50% 1 10 10 1h php-apache Deployment/php-apache 49%/50% 1 10 10 1h php-apache Deployment/php-apache 48%/50% 1 10 10 1h php-apache Deployment/php-apache 49%/50% 1 10 10 1h php-apache Deployment/php-apache 48%/50% 1 10 10 1h

kubectl get –raw “/apis/metrics.k8s.io/v1beta1/pods” jq

curl -k -X GET -H “Content-Type: application/json” -H “Accept: application/json” -H “Authorization: Bearer $token” https://10.222.16.203:8888/apis/metrics.k8s.io/v1beta1/namespaces/dex/pods/php-apache-8cf5bcb75-7bqhm

[bottle@start ~/Code/Go/src/github.com/kubernetes-incubator]$ curl -k -X GET -H “Content-Type: application/json” -H “Accept: application/json” -H “Authorization: Bearer $token” https://10.222.16.203:8888/apis/metrics.k8s.io/v1beta1/namespaces/dex/pods/php-apache-8cf5bcb75-hnqjp { “kind”: “PodMetrics”, “apiVersion”: “metrics.k8s.io/v1beta1”, “metadata”: { “name”: “php-apache-8cf5bcb75-hnqjp”, “namespace”: “dex”, “selfLink”: “/apis/metrics.k8s.io/v1beta1/namespaces/dex/pods/php-apache-8cf5bcb75-hnqjp”, “creationTimestamp”: “2018-05-24T12:17:00Z” }, “timestamp”: “2018-05-24T12:16:00Z”, “window”: “1m0s”, “containers”: [ { “name”: “php-apache”, “usage”: { “cpu”: “302m”, “memory”: “17212Ki” } } ] }⏎

cpu usage的计算方式：

kubectl get pods -w wide =》 docker ps|grep {pod} => docker inspect => docker id => cgroup dir

mount -t cgroup /sys/fs/cgroup/cpu/kubepods/burstable/pod737b66b1-5f31-11e8-a37d-ecf4bbdeea94/51755afefa67bc26ce123276a2fb9636691e402994dece373e4f8b8749db7e7d

cgroup中找cpuacct

cat cpuacct.usage; sleep 60; cat cpuacct.usage 4773394102426 ($1) 4796220173484 ($2)

( $2 - $1 ) / 60 * 1,000,000,000

####

Source descriptions:

##

@type tail path /var/log/messages pos_file /var/log/td-agent/messages.pos tag node203.*

@type syslog

</source>

@type tail path /var/log/containers/* refresh_interval 60 read_from_head true encoding, from_encoding UTF-8 pos_file /var/log/td-agent/container.pos tag node203.*

@type json time_format %Y-%m-%dT%H:%M:%S.%NZ #I0525 02:22:28.679464

</source>

####

Output descriptions:

## <match node203.**> @type kafka brokers 172.25.53.90:9092,172.25.53.91:9092,172.25.53.92:9092 default_topic “k8s” output_include_time true output_include_tag true </match>

<match node203.**> @type stdout @id output_stdout </match>

./bin/plugin install file:/opt/elasticsearch-2.3.3/elasticsearch-sql-2.3.3.0.zip

input { kafka{ bootstrap_servers => [“172.25.53.90:9092,172.25.53.91:9092,172.25.53.92:9092”] topics => [“k8s”] } }

filter { json { source => “message” } date { match => [“time”, “yyyy-MM-dd’T’HH:mm:ss.SSSZ”] } }

output { elasticsearch { hosts => [ “172.25.47.227:9200”, “172.25.47.232:9200”, “172.25.47.233:9200” ] index => “k8s-%{+YYYY.MM.dd}” } }

time_format "%Y-%m-%dT%H:%M:%S.%3NZ"
<parse>
  time_format %Y-%m-%dT%H:%M:%S.%3NZ
</parse>
<inject>
  time_format %Y-%m-%dT%H:%M:%S.%3NZ
</inject>

node88.var.log.containers.php-apache-8cf5bcb75-7bqhm_dex_php-apache-51755afefa67bc26ce123276a2fb9636691e402994dece373e4f8b8749db7e7d.log

bin/kafka-topics.sh –list –zookeeper 172.25.53.94:2181 bin/kafka-console-consumer.sh –zookeeper 172.25.53.94:2181 –topic k8s –from-beginning bin/kafka-topics.sh –describe –zookeeper 172.25.53.94:2181 –topic topic bin/kafka-topics.sh –zookeeper 172.25.53.94:2181 –delete –topic k8s

{“log”:”10.244.2.1 - - [28/May/2018:05:58:31 +0000] "GET / HTTP/1.1" 200 195 "-" "Wget"\n”,”stream”:”stdout”,”time”:”2018-05-28T05:58:31.265572495Z”,”tag”:”node88.var.log.containers.php-apache-8cf5bcb75-7bqhm_dex_php-apache-51755afefa67bc26ce123276a2fb9636691e402994dece373e4f8b8749db7e7d.log”}

{“log”:”I0528 05:58:31.032892 1 round_trippers.go:393] Accept: application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.7,text/plain;version=0.0.4;q=0.3,/;q=0.1\n”,”stream”:”stderr”,”time”:1527458311,”tag”:”node203.var.log.containers.kube-apiserver-yz-222-16-203.h.chinabank.com.cn_kube-system_kube-apiserver-23985aa7bef44c6e2232c9106fb56a5ae4e65a8d569e5a5090da787f75ec551c.log”}

        match => ["time", "yyyy-MM-ddTHH:mm:ss.SSSZ"]
                    match => ["time", "yyyy-MM-ddGHH:mm:ss.SSSZ", "ISO8601"]

[2018-05-28T15:19:53,442][WARN ][logstash.outputs.elasticsearch] Could not index event to Elasticsearch. {:status=>400, :action=>[“index”, {:_id=>nil, :_index=>”k8s”, :_type=>”doc”, :_routing=>nil}, #<LogStash::Event:0x33e884ac>], :response=>{“index”=>{“_index”=>”k8s”, “_type”=>”doc”, “_id”=>”Jo-epWMBWl5i1amZH_dX”, “status”=>400, “error”=>{“type”=>”mapper_parsing_exception”, “reason”=>”failed to parse [time]”, “caused_by”=>{“type”=>”illegal_argument_exception”, “reason”=>”For input string: "2018-05-28.07:19:52.466Z"”}}}}}

        match => ["time", "yyyy-MM-dd.HH:mm:ss.SSSZ"]


    date {
        match => ["time", "yyyy-MM-dd.HH:mm:ss.SSS"]
        target => "@timestamp"
    }

28451 Elasticsearch 14996 Logstash 14790 Logstash 15030 Jps [root@YZ-25-47-227 logstash-6.2.4]# [2018-05-28T15:41:36,006][WARN ][logstash.outputs.elasticsearch] Could not index event to Elasticsearch. {:status=>400, :action=>[“index”, {:_id=>nil, :_index=>”k8s”, :_type=>”doc”, :_routing=>nil}, #<LogStash::Event:0x46a43bc9>], :response=>{“index”=>{“_index”=>”k8s”, “_type”=>”doc”, “_id”=>”LjexpWMBSulZqxI1_yaD”, “status”=>400, “error”=>{“type”=>”mapper_parsing_exception”, “reason”=>”failed to parse [time]”, “caused_by”=>{“type”=>”illegal_argument_exception”, “reason”=>”For input string: "2018-05-28T07:41:33.210"”}}}}} [2018-05-28T15:4

[2018-05-28T15:53:19,148][WARN ][logstash.outputs.elasticsearch] Could not index event to Elasticsearch. {:status=>400, :action=>[“index”, {:_id=>nil, :_index=>”k8s”, :_type=>”doc”, :_routing=>nil}, #<LogStash::Event:0x1c5c2fd9>], :response=>{“index”=>{“_index”=>”k8s”, “_type”=>”doc”, “_id”=>”h5C8pWMBWl5i1amZur4K”, “status”=>400, “error”=>{“type”=>”mapper_parsing_exception”, “reason”=>”failed to parse [time]”, “caused_by”=>{“type”=>”illegal_argument_exception”, “reason”=>”For input string: "2018-05-28T07:45:28.778Z"”}}}}}

/api/v1/namespaces/{namespace}/pods/{name}/log

LIST curl -H “Authorization: Bearer $token” -k https://apigw.example.com:8143/clusters/system/apis/autoscaling/v1/namespaces/dex/horizontalpodautoscalers

READ curl -X GET -H “Authorization: Bearer $token” -k https://apigw.example.com:8143/clusters/system/api/v1/namespaces/dex/pods/apigw-c49c7865d-p9v6l/log?tailLines=10

POST curl -X POST -H “Content-Type: application/json” -H “Accept: application/json” -H “Authorization: Bearer $token” -k https://apigw.example.com:8143/clusters/system/apis/autoscaling/v1/namespaces/dex/horizontalpodautoscalers –data ‘{“apiVersion”:”autoscaling/v1”,”kind”:”HorizontalPodAutoscaler”,”metadata”:{“name”:”php-apache”,”namespace”:”dex”},”spec”:{“maxReplicas”:10,”minReplicas”:1,”scaleTargetRef”:{“apiVersion”:”extensions/v1beta1”,”kind”:”Deployment”,”name”:”php-apache”},”targetCPUUtilizationPercentage”:50}}’

nginx location proxy_pass 假设下面四种情况分别用 http://192.168.1.1/proxy/test.html 进行访问。

第一种： location /proxy/ { proxy_pass http://127.0.0.1/; } 代理到URL：http://127.0.0.1/test.html

第二种（相对于第一种，最后少一个 / ） location /proxy/ { proxy_pass http://127.0.0.1; } 代理到URL：http://127.0.0.1/proxy/test.html

第三种： location /proxy/ { proxy_pass http://127.0.0.1/aaa/; } 代理到URL：http://127.0.0.1/aaa/test.html

第四种（相对于第三种，最后少一个 / ） location /proxy/ { proxy_pass http://127.0.0.1/aaa; } 代理到URL：http://127.0.0.1/aaatest.html