пятница, 12 мая 2017 г.

kafka: decrease topic replication factor

Внезапно пришлось редусить кластер кафки с 5 до 3 нод. Все топики были с репликейшн фактор 3, что на 3х нодах расточительно. Просто так нельзя взять и уменьшить его, но есть прикольный, хоть и чуть геморный способ через ручное назначение партиций.
Итак, порядок примерно такой:
1. делаем json со списком топиков.
2. генерим для json с распределением партиций по нодам (можно например указать только оставшие 3 брокера, если мы еще не убили 2 оставшихся и не перенесли партиции)
3. удаляем из него по одной ноде для каждой партиции
4. скармливаем json кафке
5. проверяем, что в дескрайбе фактор стал 2.

Пример на тестовом топике, заодно выводим из работы из 3 брокеров одного:

root@kafka1:~# /opt/kafka/bin/kafka-topics.sh --zookeeper zoo1:2181/kafka --create --topic repl-test --replication-factor 3 --partitions 8
Created topic "repl-test".
root@kafka1:~# /opt/kafka/bin/kafka-topics.sh --zookeeper zoo1:2181/kafka --describe --topic repl-test
Topic:repl-test PartitionCount:8 ReplicationFactor:3 Configs:
Topic: repl-test Partition: 0 Leader: 3 Replicas: 3,1,2 Isr: 3,1,2
Topic: repl-test Partition: 1 Leader: 1 Replicas: 1,2,3 Isr: 1,2,3
Topic: repl-test Partition: 2 Leader: 2 Replicas: 2,3,1 Isr: 2,3,1
Topic: repl-test Partition: 3 Leader: 3 Replicas: 3,2,1 Isr: 3,2,1
Topic: repl-test Partition: 4 Leader: 1 Replicas: 1,3,2 Isr: 1,3,2
Topic: repl-test Partition: 5 Leader: 2 Replicas: 2,1,3 Isr: 2,1,3
Topic: repl-test Partition: 6 Leader: 3 Replicas: 3,1,2 Isr: 3,1,2
Topic: repl-test Partition: 7 Leader: 1 Replicas: 1,2,3 Isr: 1,2,3
root@kafka1:~# vi topic.json
root@kafka1:~# cat topic.json

{"topics":
[{"topic": "repl-test"}],
"version":1
}
root@kafka1:~# /opt/kafka/bin/kafka-reassign-partitions.sh --zookeeper zoo1:2181/kafka --topics-to-move-json-file topic.json --broker-list "1,2,3" --generate
Current partition replica assignment

{"version":1,"partitions":[{"topic":"repl-test","partition":2,"replicas":[2,3,1]},{"topic":"repl-test","partition":7,"replicas":[1,2,3]},{"topic":"repl-test","partition":5,"replicas":[2,1,3]},{"topic":"repl-test","partition":3,"replicas":[3,2,1]},{"topic":"repl-test","partition":0,"replicas":[3,1,2]},{"topic":"repl-test","partition":4,"replicas":[1,3,2]},{"topic":"repl-test","partition":1,"replicas":[1,2,3]},{"topic":"repl-test","partition":6,"replicas":[3,1,2]}]}
Proposed partition reassignment configuration

{"version":1,"partitions":[{"topic":"repl-test","partition":2,"replicas":[2,3,1]},{"topic":"repl-test","partition":7,"replicas":[1,2,3]},{"topic":"repl-test","partition":5,"replicas":[2,1,3]},{"topic":"repl-test","partition":3,"replicas":[3,2,1]},{"topic":"repl-test","partition":0,"replicas":[3,1,2]},{"topic":"repl-test","partition":4,"replicas":[1,3,2]},{"topic":"repl-test","partition":1,"replicas":[1,2,3]},{"topic":"repl-test","partition":6,"replicas":[3,1,2]}]}
root@kafka1:~# vi tmppart.json
root@kafka1:~# cat tmppart.json

{"version":1,"partitions":[{"topic":"repl-test","partition":2,"replicas":[2,3,1]},{"topic":"repl-test","partition":7,"replicas":[1,2,3]},{"topic":"repl-test","partition":5,"replicas":[2,1,3]},{"topic":"repl-test","partition":3,"replicas":[3,2,1]},{"topic":"repl-test","partition":0,"replicas":[3,1,2]},{"topic":"repl-test","partition":4,"replicas":[1,3,2]},{"topic":"repl-test","partition":1,"replicas":[1,2,3]},{"topic":"repl-test","partition":6,"replicas":[3,1,2]}]}                                                                                                  
root@kafka1:~# cat tmppart.json| python -m json.tool > part_cur.json root@kafka1:~# cat part_cur.json                                                
{                                                                                                                                                                                           
"partitions": [                                                                                                                                                                             
{                                                                                                                                                                                           
"partition": 2,                                                                                                                                                                             
"replicas": [                                                                                                                                                                               
2,                                                                                                                                                                                          
3,                                                                                                                                                                                          
1                                                                                                                                                                                           
],                                                                                                                                                                                          
"topic": "repl-test"                                                                                                                                                                        
},                                                                                                                                                                                          
{                                                                                                                                                                                           
"partition": 7,                                                                                                                                                                             
"replicas": [                                                                                                                                                                               
1,                                                                                                                                                                                          
2,
3
],
"topic": "repl-test"
},
{
"partition": 5,
"replicas": [
2,
1,
3
],
"topic": "repl-test"
},
{
"partition": 3,
"replicas": [
3,
2,
1
],
"topic": "repl-test"
},
{
"partition": 0,
"replicas": [
3,
1,
2
],
"topic": "repl-test"
},
{
"partition": 4,
"replicas": [
1,
3,
2
],
"topic": "repl-test"
},
{
"partition": 1,
"replicas": [
1,
2,
3
],
"topic": "repl-test"
},
{
"partition": 6,
"replicas": [
3,
1,
2
],
"topic": "repl-test"
}
],
"version": 1
}
root@kafka1:~# cp part_cur.json part_new.json
root@kafka1:~# vi part_new.json
root@kafka1:~# cat part_new.json

{
"partitions": [
{
"partition": 2,
"replicas": [
2,
1
],
"topic": "repl-test"
},
{
"partition": 7,
"replicas": [
1,
2
],
"topic": "repl-test"
},
{
"partition": 5,
"replicas": [
2,
1
],
"topic": "repl-test"
},
{
"partition": 3,
"replicas": [
2,
1
],
"topic": "repl-test"
},
{
"partition": 0,
"replicas": [
1,
2
],
"topic": "repl-test"
},
{
"partition": 4,
"replicas": [
1,
2
],
"topic": "repl-test"
},
{
"partition": 1,
"replicas": [
1,
2
],
"topic": "repl-test"
},
{
"partition": 6,
"replicas": [
1,
2
],
"topic": "repl-test"
}
],
"version": 1
}
root@kafka1:~# /opt/kafka/bin/kafka-reassign-partitions.sh --zookeeper zoo1:2181/kafka --reassignment-json-file part_new.json --verify
Status of partition reassignment:
ERROR: Assigned replicas (3,1,2) don't match the list of replicas for reassignment (1,2) for partition [repl-test,0]
ERROR: Assigned replicas (1,3,2) don't match the list of replicas for reassignment (1,2) for partition [repl-test,4]
ERROR: Assigned replicas (2,3,1) don't match the list of replicas for reassignment (2,1) for partition [repl-test,2]
ERROR: Assigned replicas (1,2,3) don't match the list of replicas for reassignment (1,2) for partition [repl-test,7]
ERROR: Assigned replicas (3,1,2) don't match the list of replicas for reassignment (1,2) for partition [repl-test,6]
ERROR: Assigned replicas (2,1,3) don't match the list of replicas for reassignment (2,1) for partition [repl-test,5]
ERROR: Assigned replicas (3,2,1) don't match the list of replicas for reassignment (2,1) for partition [repl-test,3]
ERROR: Assigned replicas (1,2,3) don't match the list of replicas for reassignment (1,2) for partition [repl-test,1]
Reassignment of partition [repl-test,0] failed
Reassignment of partition [repl-test,4] failed
Reassignment of partition [repl-test,2] failed
Reassignment of partition [repl-test,7] failed
Reassignment of partition [repl-test,6] failed
Reassignment of partition [repl-test,5] failed
Reassignment of partition [repl-test,3] failed
Reassignment of partition [repl-test,1] failed
root@kafka1:~# ((
> ^C
root@kafka1:~# /opt/kafka/bin/kafka-topics.sh --zookeeper zoo1:2181/kafka --describe --topic repl-test
Topic:repl-test PartitionCount:8 ReplicationFactor:3 Configs:
Topic: repl-test Partition: 0 Leader: 3 Replicas: 3,1,2 Isr: 3,1,2
Topic: repl-test Partition: 1 Leader: 1 Replicas: 1,2,3 Isr: 1,2,3
Topic: repl-test Partition: 2 Leader: 2 Replicas: 2,3,1 Isr: 2,3,1
Topic: repl-test Partition: 3 Leader: 3 Replicas: 3,2,1 Isr: 3,2,1
Topic: repl-test Partition: 4 Leader: 1 Replicas: 1,3,2 Isr: 1,3,2
Topic: repl-test Partition: 5 Leader: 2 Replicas: 2,1,3 Isr: 2,1,3
Topic: repl-test Partition: 6 Leader: 3 Replicas: 3,1,2 Isr: 3,1,2
Topic: repl-test Partition: 7 Leader: 1 Replicas: 1,2,3 Isr: 1,2,3
root@kafka1:~# /opt/kafka/bin/kafka-reassign-partitions.sh --zookeeper zoo1:2181/kafka --reassignment-json-file part_new.json --execute
Current partition replica assignment


{"version":1,"partitions":[{"topic":"repl-test","partition":2,"replicas":[2,3,1]},{"topic":"repl-test","partition":7,"replicas":[1,2,3]},{"topic":"repl-test","partition":5,"replicas":[2,1,3]},{"topic":"repl-test","partition":3,"replicas":[3,2,1]},{"topic":"repl-test","partition":0,"replicas":[3,1,2]},{"topic":"repl-test","partition":4,"replicas":[1,3,2]},{"topic":"repl-test","partition":1,"replicas":[1,2,3]},{"topic":"repl-test","partition":6,"replicas":[3,1,2]}]}


Save this to use as the --reassignment-json-file option during rollback
Successfully started reassignment of partitions {"version":1,"partitions":[{"topic":"repl-test","partition":0,"replicas":[1,2]},{"topic":"repl-test","partition":4,"replicas":[1,2]},{"topic":"repl-test","partition":2,"replicas":[2,1]},{"topic":"repl-test","partition":7,"replicas":[1,2]},{"topic":"repl-test","partition":6,"replicas":[1,2]},{"topic":"repl-test","partition":5,"replicas":[2,1]},{"topic":"repl-test","partition":3,"replicas":[2,1]},{"topic":"repl-test","partition":1,"replicas":[1,2]}]}
root@kafka1:~# /opt/kafka/bin/kafka-topics.sh --zookeeper zoo1:2181/kafka --describe --topic repl-test
Topic:repl-test PartitionCount:8 ReplicationFactor:2 Configs:
Topic: repl-test Partition: 0 Leader: 1 Replicas: 1,2 Isr: 1,2
Topic: repl-test Partition: 1 Leader: 1 Replicas: 1,2 Isr: 1,2
Topic: repl-test Partition: 2 Leader: 2 Replicas: 2,1 Isr: 2,1
Topic: repl-test Partition: 3 Leader: 2 Replicas: 2,1 Isr: 2,1
Topic: repl-test Partition: 4 Leader: 1 Replicas: 1,2 Isr: 1,2
Topic: repl-test Partition: 5 Leader: 2 Replicas: 2,1 Isr: 2,1
Topic: repl-test Partition: 6 Leader: 1 Replicas: 1,2 Isr: 1,2
Topic: repl-test Partition: 7 Leader: 1 Replicas: 1,2 Isr: 1,2