From bf5ae9cf8a131eb69a07b1dffdb8b01af8240579 Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Fri, 11 Sep 2020 10:31:47 +0200 Subject: [PATCH] Fix #3299: Build KenLM on CI --- .gitmodules | 3 + data/smoke_test/vocab.txt | 501 ++++++++++++++++++ kenlm | 1 + native_client/BUILD | 7 +- taskcluster/.shared.yml | 32 ++ taskcluster/kenlm_android-arm64-cpu-opt.yml | 17 + taskcluster/kenlm_android-armv7-cpu-opt.yml | 17 + taskcluster/kenlm_android-x86_64-cpu-opt.yml | 17 + taskcluster/kenlm_darwin-amd64-cpu-opt.yml | 13 + taskcluster/kenlm_linux-amd64-cpu-opt.yml | 16 + taskcluster/kenlm_linux-arm64-cpu-opt.yml | 16 + taskcluster/kenlm_linux-rpi3-cpu-opt.yml | 16 + .../kenlm_multistrap_arm64_buster.conf | 14 + taskcluster/kenlm_multistrap_rpi3_buster.conf | 14 + taskcluster/kenlm_tc-build.sh | 93 ++++ taskcluster/kenlm_tc-package.sh | 14 + taskcluster/kenlm_tc-setup.sh | 109 ++++ .../kenlm_win-amd64-cpu-opt.yml.DISABLED | 12 + taskcluster/tc-all-utils.sh | 22 +- taskcluster/tc-all-vars.sh | 1 + taskcluster/tc-android-ds-tests.sh | 8 - taskcluster/tc-android-utils.sh | 4 +- taskcluster/tc-cpp_tflite-tests-prod.sh | 1 - taskcluster/tc-cpp_tflite_basic-ds-tests.sh | 1 - taskcluster/tc-node_tflite-tests-prod.sh | 1 - taskcluster/tc-py-utils.sh | 7 +- taskcluster/tc-python_tflite-tests-prod.sh | 4 - taskcluster/tc-python_tflite-tests.sh | 4 - taskcluster/tc-scorer-tests.sh | 109 ++++ taskcluster/test-android-opt-base.tyml | 4 +- taskcluster/test-armbian-opt-base.tyml | 1 - taskcluster/test-darwin-opt-base.tyml | 2 - ...t-generate_scorer-android-24-arm64-opt.yml | 15 + ...t-generate_scorer-android-24-armv7-opt.yml | 15 + ...-generate_scorer-android-24-x86_64-opt.yml | 15 + ...t-generate_scorer-android-25-arm64-opt.yml | 15 + ...t-generate_scorer-android-25-armv7-opt.yml | 15 + ...-generate_scorer-android-25-x86_64-opt.yml | 15 + ...-generate_scorer-android-26-x86_64-opt.yml | 15 + ...-generate_scorer-android-28-x86_64-opt.yml | 15 + ...-generate_scorer-android-29-x86_64-opt.yml | 15 + ...-generate_scorer-android-30-x86_64-opt.yml | 15 + .../test-generate_scorer-darwin-amd64-opt.yml | 11 + .../test-generate_scorer-linux-amd64-opt.yml | 11 + .../test-generate_scorer-linux-arm64-opt.yml | 14 + .../test-generate_scorer-linux-rpi3-opt.yml | 14 + taskcluster/test-linux-opt-base.tyml | 2 - taskcluster/test-linux-opt-tag-base.tyml | 2 - taskcluster/test-raspbian-opt-base.tyml | 1 - taskcluster/test-win-cuda-opt-base.tyml | 1 - taskcluster/test-win-opt-base.tyml | 1 - 51 files changed, 1246 insertions(+), 42 deletions(-) create mode 100644 data/smoke_test/vocab.txt create mode 160000 kenlm create mode 100644 taskcluster/kenlm_android-arm64-cpu-opt.yml create mode 100644 taskcluster/kenlm_android-armv7-cpu-opt.yml create mode 100644 taskcluster/kenlm_android-x86_64-cpu-opt.yml create mode 100644 taskcluster/kenlm_darwin-amd64-cpu-opt.yml create mode 100644 taskcluster/kenlm_linux-amd64-cpu-opt.yml create mode 100644 taskcluster/kenlm_linux-arm64-cpu-opt.yml create mode 100644 taskcluster/kenlm_linux-rpi3-cpu-opt.yml create mode 100644 taskcluster/kenlm_multistrap_arm64_buster.conf create mode 100644 taskcluster/kenlm_multistrap_rpi3_buster.conf create mode 100755 taskcluster/kenlm_tc-build.sh create mode 100755 taskcluster/kenlm_tc-package.sh create mode 100755 taskcluster/kenlm_tc-setup.sh create mode 100644 taskcluster/kenlm_win-amd64-cpu-opt.yml.DISABLED create mode 100644 taskcluster/tc-scorer-tests.sh create mode 100644 taskcluster/test-generate_scorer-android-24-arm64-opt.yml create mode 100644 taskcluster/test-generate_scorer-android-24-armv7-opt.yml create mode 100644 taskcluster/test-generate_scorer-android-24-x86_64-opt.yml create mode 100644 taskcluster/test-generate_scorer-android-25-arm64-opt.yml create mode 100644 taskcluster/test-generate_scorer-android-25-armv7-opt.yml create mode 100644 taskcluster/test-generate_scorer-android-25-x86_64-opt.yml create mode 100644 taskcluster/test-generate_scorer-android-26-x86_64-opt.yml create mode 100644 taskcluster/test-generate_scorer-android-28-x86_64-opt.yml create mode 100644 taskcluster/test-generate_scorer-android-29-x86_64-opt.yml create mode 100644 taskcluster/test-generate_scorer-android-30-x86_64-opt.yml create mode 100644 taskcluster/test-generate_scorer-darwin-amd64-opt.yml create mode 100644 taskcluster/test-generate_scorer-linux-amd64-opt.yml create mode 100644 taskcluster/test-generate_scorer-linux-arm64-opt.yml create mode 100644 taskcluster/test-generate_scorer-linux-rpi3-opt.yml diff --git a/.gitmodules b/.gitmodules index 70cd92a1..3ec0cd1b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -5,3 +5,6 @@ [submodule "tensorflow"] path = tensorflow url = https://github.com/mozilla/tensorflow.git +[submodule "kenlm"] + path = kenlm + url = https://github.com/kpu/kenlm diff --git a/data/smoke_test/vocab.txt b/data/smoke_test/vocab.txt new file mode 100644 index 00000000..00afee90 --- /dev/null +++ b/data/smoke_test/vocab.txt @@ -0,0 +1,501 @@ + she had your dark suit in greasy wash water all year +groups we were brought together with several other victims families when i saw aicha in the media +coming over when her son was indicted and i thought what a brave woman someday i want to meet that woman when im stronger i was still in deep grief i knew i didnt have the strength i knew i would find her someday or we would find each other because when people heard that my son +was a victim i got immediate sympathy +but when people learned what her son was accused of she didnt get that sympathy but her suffering is equal to mine so we met in november two thousand and two and aicha will now tell you how that came about +today because of +introduced me to five families and i saw phyllis and i watched +and i saw in her eyes that she was a mother just like me +i was married when i was fourteen i lost a child when i was fifteen a second child when i was sixteen so the story with zacarias was too much really +so thats why i decided to tell my story so that my suffering is something positive for other women +all the women all the mothers +i first learned that my son had been in the world trade center on the morning of september eleventh two thousand and one +its up to us women because we are women because we love our children +its not against women its for us for us women for +i talk against violence against terrorism i go to schools to talk to young muslim +girls so they dont accept to be married against their will very young +so if i can save one of the young girls and avoid that they get married and suffer as much as i did well this is something +i have learned so much +we didnt know if he had perished yet until thirty six hours later at the time +family members but we were all so nervous why does she want to meet us +and then she was nervous why did we want to meet her what did we want from each other +before we knew each others names or anything we had embraced +and wept then we sat in a circle +with support with help from people experienced in this kind of reconciliation and aicha started and she said +i dont know if my son is guilty or innocent but i want to tell you how sorry i am for what happened to your families +i know what it is to suffer +and i feel that if there is a crime a person should be tried fairly and punished +but +she reached out to us in that way and it was id like to say it was an ice breaker and what happened then is we all told our stories +and we all connected as human beings by the end of the afternoon it was about three hours after lunch +wed felt as if wed known each other forever now what i learned from her is a woman not only who could be so generous under these present circumstances and what it was then and what was being done to her son but the life shes had i never had met +someone with such a hard life from such a totally different culture and environment from my own +we knew that it was political +being afraid of the other but making that step +and then realizing hey this wasnt so hard who else can i meet that i dont know or that im so different from +so aicha do you have +a couple of words for conclusion because our time is up +we were afraid of what our country was going to do in the name of our son my husband orlando and i +i wanted to say that we have to try to know other people the other +and i hope that someday well all live together in peace and respecting each other this is what i wanted to say +and our family and when i saw it and yet through the shock +the terrible shock and the terrible +explosion in our lives literally +we were not vengeful +on six counts of conspiracy to commit terrorism +and the u s government called for a death penalty for him if convicted +my husband and i spoke out +in opposition to that publicly through that and through human rights +ive also had some meals that make me want to dry heave so its about choosing the parts of the bible about compassion about tolerance about loving your neighbor as opposed to the parts +about homosexuality is a sin or intolerance or violence which are very much in the bible as well +so if we are to find any meaning in this book then we have to really engage it and wrestle with it and i thought id end with just a couple more +theres me reading the bible thats how i hailed taxi cabs +morning but it served well for a day so anyway thank you so much for letting me +so +and it was about the year i spent reading the encyclopedia britannica from a to z in my quest to learn everything in the world or more precisely from +which is a type of east asian music all the way to zwyiec which is well i dont +although listening to kevin kelly you dont have to remember anything you can just google it so i wasted some time there +i love those experiments but i think that the most profound and life changing experiment that ive done is my most recent experiment +i thought id tell you a little about what i like to write and i like to immerse myself in my topics i just like to dive right in and become sort of a human guinea pig and +where i spent a year trying to follow all of the rules of the bible the year of living biblically and +i undertook this for two reasons the first was that i grew up with no religion at all as i say in my book im jewish in the same way the olive garden is italian +so +but ive become increasingly interested in religion i do think its the defining issue of our time or one of the main ones and i have a son i want to know what to teach him so i decided to dive in head first and try to live the bible +the second reason i undertook this is because im concerned about the rise of fundamentalism religious fundamentalism and people who say +what if you really did take the bible literally i decided to take it to its logical conclusion and take everything in the bible literally +without picking and choosing the first thing i did was i got a stack of bibles i had christian bibles i had +jewish bibles a friend of mine sent me something called a hip hop bible where the twenty three rd psalm is rendered as the lord is all that as opposed to what i knew it as the lord is my shepherd +then i went down and i read several versions and i wrote down every single law that i could find and this was a very long list over seven hundred rules +and they range from the famous ones that i had heard of the ten commandments love your neighbor be fruitful and multiply so i wanted to follow those and actually i take my projects very seriously because i had twins during my year so i +definitely take my projects seriously but i also wanted to follow the hundreds of arcane and obscure laws that are in the bible +there is the law in leviticus you cannot shave the corners of your beard i didnt know where my corners were so i decided +to let the whole thing grow and this is what i looked like by the end as you can imagine i spent a lot of time at airport security +my wife wouldnt kiss me for the last two months so certainly the challenge was there the bible says you cannot wear clothes made of mixed fibers so i thought sounds strange but ill try it you only know +i see my life as a series of experiments so i work for esquire magazine and a couple of years ago i wrote an article called my outsourced life +i got rid of all my poly cotton t shirts the bible says that if two men are in a fight and the wife of one of those men grabs the testicles of the other +her hand shall be cut off so i wanted to follow that rule +wife was standing nearby looking like she had a strong grip so +theres another shot of my beard i will say it was an amazing year because it really was life changing and incredibly challenging and there were two types of laws +were particularly challenging the first was avoiding the little sins that we all commit every day +know i could spend a year not killing but spending a year not gossiping not coveting not lying you know i live in new york and i work as a journalist so this was seventy five eighty percent of my +but it was really interesting because i was able to make some progress because i couldnt believe how +my behavior changed my thoughts this was one of the huge lessons of the year is that i almost pretended to be a better person and i became a little bit of a better person so +i had always thought you know you change your mind and you change your behavior but its often the other way round you change your behavior +and you change your mind so you know if you want to become more compassionate you visit sick people in the hospital and you will become more compassionate +where i hired a team of people in bangalore india to live my life for me so they answered my emails they answered my phone they argued with my wife for me and they +you donate money to a cause and you become emotionally involved in that cause so it really was cognitive psychology +that if you smile you will become happier which as we know is actually true the second type of +rule that was difficult to obey was the rules that will get you into a little trouble in twenty one st century america and +the clearest example of this is stoning adulterers +but its a big part of the bible so i +had to address +i was able to stone one adulterer it happened i was in the park and i was dressed in my biblical clothing sandals and a white robe you know because again the outer +see how dressing biblically affected my mind +up to me and he said why are you dressed like that and i explained my project and he said well i am an adulterer are you going to stone me and i said well that would be great +and +i took out a handful of stones from my pocket that i had been carrying around for weeks hoping for just this interaction and you know they were pebbles +out of my hand he was actually an elderly man mid seventies just so you know but hes still an adulterer and still quite angry he grabbed them out of my hand and threw them at my face +and i felt that i could eye for an eye i could retaliate and throw one back at him so that was my experience stoning and it did allow me to talk +about in a more serious way these big issues how can the bible be so barbaric in some places and yet so incredibly wise in others +it has all of these authors and editors over hundreds of years and its sort of evolved its not a book that was written and came down from on high +my son bedtime stories it was the best month of my life because i just sat back and i read books and watched movies +so i thought i would end by telling you just a couple of the take away the bigger +lessons that i learned from my year the first is thou shalt not take the bible literally this +very very clear early on because if you do then you end up acting like a crazy person and stoning adulterers or here +well thats another i did spend some time shepherding its a very relaxing vocation i recommend it but this one is +and my wife thought this was very offensive so she sat in every seat in our apartment and i had to spend much of the year standing until +i bought my own seat and carried it around +so you know i met with creationists i went to the creationists museum and these are the ultimate literalists and it was fascinating because they were not stupid people at all +that they distort all the data to fit their model and they go through these amazing mental gymnastics to accomplish this and i will say though +was a wonderful experience more recently i wrote an article for esquire called about radical honesty and this is a movement +the museum is gorgeous they really did a fantastic job if youre ever in kentucky theres +i think its crazy they did a great job +another lesson is that +thou shalt give thanks and this one was a big lesson because i was praying giving these prayers of thanksgiving which was odd for an agnostic but +saying thanks all the time every day and i started to change my perspective and i started to realize the hundreds of little things that go right every day +that i didnt even notice that i took for granted as opposed to focusing on the three or four that went wrong +so this is actually a key to happiness for me is to just remember when i came over here the car didnt flip over and i didnt trip coming up the stairs its a remarkable thing +this one was unexpected because i started the year as an agnostic and by the end of the year i became what a friend of mine calls a reverent agnostic which i love +a movement so if anyone wants to join the basic idea is whether or not there is a god theres something important and beautiful about the idea of sacredness and that our rituals can be sacred the sabbath can be +this is started by a psychologist in virginia who says that you should never ever lie except maybe during poker and golf his only exceptions and more than that +this was one of the great things about my year doing the sabbath because i am a workaholic so having this one day where you cannot work it really that changed my life +journey i wanted it to be about religion in america so i spent time with evangelical christians and hasidic jews and the +im very proud because i think im the only person in america to out bible talk a jehovahs witness +thank you +but it was +because i had some very preconceived notions about for instance evangelical christianity and i found that its such a wide +and varied movement that it is difficult to make generalizations about it theres a group i met with called the red letter christians and they focus on +words in the bible which are the ones that jesus spoke thats how they printed them in the old bibles and +is that jesus never talked about homosexuality they have a pamphlet that says heres what jesus said about homosexuality and you open it up and theres nothing in it so +they say jesus did talk a lot about helping the outcasts helping poor people so this was very inspiring to me +i recommend jim wallace and tony campolo theyre very inspiring leaders even though i disagree with much of what they say also thou shalt not +i was shocked learning how much of my life is governed by irrational forces and +the thing is if theyre not harmful theyre not to be completely dismissed because i learned that i was thinking i was doing all these rituals these biblical rituals separating my +and linen and i would ask these religious people why would the bible possibly tell us to do this why would god care and they said we dont know but its just rituals +that give us meaning and i would say but thats crazy and they would say well what about you you blow out candles on top of a birthday cake if a guy from mars came down and saw +heres one guy blowing out the fire on top of a cake versus another guy not wearing clothes of mixed fabrics would the martians say well that +he makes sense but that guys crazy so no i think that +are not harmful but rituals by themselves are not to be dismissed and finally +i learned that thou shall pick and choose and this one i learned because i tried to follow everything in the bible and +i do not recommend this at all to give you a sense of the experience the article was called i think youre fat +i failed miserably because you cant you have to pick and choose and anyone who follows the bible is going to be picking and choosing the key is +to pick and choose the right parts theres the phrase called +my argument is whats wrong with cafeterias ive had some great meals at cafeterias +and theres the sheep now the final part of the trilogy was i wanted to focus on the body and try to be the healthiest person i could be the healthiest person alive so thats what ive been doing the last couple of years +last decade subjecting myself to pain and humiliation hopefully for a good cause which is self improvement +and i just finished a couple of months ago and i have to say thank god because living so healthily was killing me +it was so overwhelming because the amount of things you have to do its just +mind boggling i was listening to all the experts and talking to sort of a board of medical advisers +and they were telling me all the things i had to do i had to eat right exercise meditate pet dogs because that lowers the blood pressure i wrote the book on a treadmill and it took me about a thousand miles to write the book +went into sunscreen i was like a glazed doughnut for most of the year +that i should also wipe down all of the remote controls and iphones in my house because those are just orgies of germs so that +and ive done this in three parts so first i started with the mind and i decided to try to get smarter by reading the entire encyclopedia britannica from a to z or more precisely from +now its a little extreme i admit but if you think about this this is actually the freakonomics authors wrote about this that more people die on a per mile basis from drunk walking than from drunk driving +so something to think about tonight if youve had a couple +so i finished and it was a success +so i finished and i +without the sex part because i have three young kids so that wasnt happening but +and i finally +have stabilized so now im back to +adopting many not all i dont wear a helmet anymore but dozens of healthy behaviors that i adopted during my year it was really a life changing project and i of course dont have time to go into all of them let me just tell you two really quickly +the first is and this was surprising to me i didnt expect this to come out but i live a much quieter life now +and this is a real underestimated under appreciated health hazard not just because it harms our hearing which it obviously does but it actually initiates the fight or flight response a loud noise will get your fight or flight response going and this +over the years can cause real damage cardiovascular damage +the world health organization just did a big study that they published this year and it was done in europe and they estimated that one point six million years of healthy living are lost +every year in europe because of noise pollution so they think its actually very deadly and by the way its also terrible for your brain +they put dirt all over the cobblestones outside the hall so that they could concentrate so without noise reduction technology our country would not exist so as a patriot i felt it was important to i wear all the earplugs and the earphones +that joy is so important to your health that very few of these behaviors will stick with me unless theres some sense of pleasure and joy in them and just to give you one instance of this food +but i think we can use their techniques and apply them to healthy food to give just one example we love crunchiness mouthfeel so i basically have tried to incorporate crunchiness into a lot of my recipes throw in some sunflower seeds +and you can almost trick yourself into thinking youre eating doritos laughter and +it had its downsides +the +because leviticus says you cannot shave so this is what i looked like by the end +thank you for that reaction laughter i look a little like moses or ted kaczynski i got both of them so there was the topiary there +we have indeed taken the best part of the meat so lets look today at a set of photographs of a people who lost so that we could gain +and know that when you see these peoples faces that these are not just images of the lakota they stand for all indigenous people +on this piece of paper is the history the way i learned it from my lakota friends and family +im here today to show my photographs of the lakota many of you may have heard of the lakota or at least the larger group of tribes called +sixty six the beginning of the transcontinental railroad a new era +we appropriated land for trails and trains to shortcut through the heart of the lakota nation the treaties were out the window in response three tribes led by the lakota chief red cloud +attacked and defeated the u s army many times over i want to repeat that part the lakota defeat the u s army +sixty eight the second fort laramie treaty clearly guarantees the sovereignty of the great sioux nation and the lakotas ownership of the sacred black hills +the lakota are one of many tribes that were moved off their land to prisoner of war camps now called reservations the pine ridge reservation +seventy one the indian appropriation act makes all indians wards of the federal government in addition the military issued orders forbidding western indians from leaving reservations +the move destroyed the reservations making it easier to further subdivide and to sell with every passing generation most of the surplus land +i believe to be the most important in this slide show this is the year of the wounded knee massacre +to this day this is the most medals of honor ever awarded for a single battle +more medals of honor were given for the indiscriminate slaughter of women and children than for any battle in world war one world war two korea vietnam iraq or afghanistan +now if any of you have ever heard of aim the american indian movement or of russell means or leonard peltier or of the stand off at oglala +the wounded knee massacre is considered the end of the indian wars whenever i visit the site of the mass grave at wounded knee +i see it not just a grave for the lakota or for the sioux but as a grave for all indigenous peoples +the holy man black elk said i did not know then how much was ended when i look back now from this high hill of my old age +i can still see the butchered women and children lying heaped and scattered all along the crooked gulch +when i saw them with eyes still young +and i can see that something else died there in the bloody mud and was buried in the blizzard +a peoples dream died there and it was a beautiful dream with +this event a new era in native american history began everything can be measured before wounded knee and after because it was in this moment with the fingers on the triggers of the hotchkiss guns +the court determined that when the sioux were resettled onto reservations and seven million acres of their land were opened up to prospectors and homesteaders the terms of the second fort laramie treaty had been violated +the court stated that the black hills were illegally taken and that the initial offering price plus interest should be paid to the sioux nation +ten statistics about native population today more than a century after the massacre at wounded knee reveal the legacy of colonization forced migration and treaty violations +at least sixty percent of the homes on the reservation are infested with black mold more than ninety percent of the population lives below the federal poverty line +the tuberculosis rate on pine ridge is approximately eight times higher than the u s national average the infant mortality rate is the highest on this continent and is about three times higher than the u s national average +the last chapter in any successful genocide is the one in which the oppressor can remove their hands and say +my god what are these people doing to themselves theyre killing each other theyre killing themselves while we watch them die +this is how we came to own these united states this is the legacy of manifest destiny prisoners are still born into prisoner of war camps long after the guards are gone +these are the bones left after the best meat has been has been taken +a long time ago a series of events was set in motion by a people who look like me by wasichu eager to take the land and the water and the gold in the hills +those events led to a domino effect that has yet to end as removed as we the dominant society may feel +what is the connection between these images of suffering and the history that i just read to you and how much of this history do you need to own even is any of this your responsibility today +been told that there must be something we can do there must be some call to action because for so long ive been standing on the sidelines +content to be a witness just taking photographs because the solution seems so far in the past i needed nothing short of a time machine to access them +the suffering of indigenous peoples is not a simple issue to fix its not something everyone can get behind the way they get behind helping haiti of ending aids or fighting a famine +and invited me again and again over five years but on pine ridge i will always be what is called wasichu and wasichu is a lakota word +the fix as its called may be much more difficult for the dominant society than say a fifty dollar check +or church trip to paint some graffiti covered houses or a suburban family donating a box of clothes they dont even want anymore +so where does that leave us shrugging our shoulders in the dark +the call to action i offer today my ted wish is this honor the treaties give back the black hills its not your business what they do with them +that means non indian but another version of this word means the one who takes the best meat for himself and thats what i want to focus on the one who takes the best part of the meat it means greedy +you can also toggle between altitude for model and manufacturer see again the diversity +and you can scroll around and see some of the different airports and the different patterns that they have this is scrolling up the east coast you can see some of the chaos thats happening in new york with the air traffic controllers having to deal with +so zooming back out real quick we see again the u s you get florida down in the right hand corner moving across to the west coast you see san francisco and los angeles big low traffic zones across nevada and arizona and thats us down there in l a and long beach on the bottom +i started taking a look as well at different perimeters because you can choose what you want to pull out from the data this is looking at ascending versus descending flights and you can see over time the ways the airports change you see the holding patterns that start to develop in the bottom of the screen and you can see eventually the airport actually flips directions +data can actually make us more human were collecting and creating all kinds of data about how were living our lives +so this is another project that i worked on with the sensible cities lab at mit this is visualizing international communications so its how new york communicates with other international cities and we set this up as a live globe in the museum of modern art in new york for the design the elastic mind exhibition +its visualizing sms messages being sent in the city of amsterdam so youre seeing the daily ebb and flow of people sending sms messages from different parts of the city until we approach new years eve where everybody says happy new year +and its enabling us to tell some amazing stories recently a wise media theorist tweeted the nineteenth century culture was defined by the novel the twentieth century culture was defined by the cinema and the culture of the twenty first century will be defined by the interface +and then youre going to see people start to gather in the center of the city to celebrate the night before which happens right here and then you can see people celebrating the next day and you can pause it and step back and forth and see different phases +so now on to something completely different some of you may recognize this this is baron wolfgang von kempelens mechanical chess playing machine and its this amazing robot that plays chess extremely well except for one thing its not a robot at all theres actually a legless man that sits in that box and controls this chess player +this was the inspiration for a web service by amazon called the mechanical turk named after this guy and its based on the premise that there are certain things that are easy for people but really difficult for computers +so they made this web service and said any programmer can write a piece of software and tap into the minds of thousands of people the nerdy side of me thought wow this is amazing i can tap into thousands of peoples minds and the other nerdy side of me thought this is horrible this is completely bizarre what does this mean for the future of mankind +so i created this drawing tool i asked people to draw a sheep facing to the left and i said ill pay you two cents for your contribution +and i started collecting sheep and i collected +a lot a lot of different sheep +lots of sheep +i took the first ten thousand sheep that i collected and i put them on a website called thesheepmarket com +where you can actually buy collections of twenty sheep you cant pick individual sheep but you can buy a single plate block of stamps as a commodity and juxtaposed against this grid you see actually by rolling over each individual one the humanity behind this hugely mechanical process +so heres a few statistics from the project approximate collection rate of eleven sheep per hour which would make a working wage of sixty nine cents per hour +there were six hundred and sixty two rejected sheep that didnt meet sheep like criteria and were thrown out of +and i believe this is going to prove true our lives are being driven by data and the presentation of that data is an opportunity for us to make some amazing interfaces that tell great stories so im going to show you a few of the projects that ive been working on over the last couple years that reflect on our lives and our systems +the flock laughter the amount of time spent drawing ranged from four seconds to forty six minutes that gives you an idea of the different types of motivations and dedication and there were seven thousand five hundred and ninety nine people that contributed to the project or were unique ip addresses so about how many people contributed but only one of them out of the seven thousand five hundred and ninety nine said this +obviously we think of sheep as followers and theres this reference to le petit prince where the narrator asks the prince to draw a sheep he draws sheep after sheep the narrators only appeased when he draws a box and he says its not about a scientific rendering of a sheep its about your own interpretation and doing something different and i like that +so there were no longer shoe makers but now there are people slapping soles on peoples shoes and the whole idea of ones relationship to their work changed a lot so i thought this was an interesting clip to divide into sixteen pieces and feed into the mechanical turk with a drawing tool +this basically allowed what you see on the left side is the original frame and on the right side you see that frame as interpreted by sixteen people who have no idea what it is theyre doing +and this was the inspiration for a project that i worked on with my friend takashi kawashima we decided to use the mechanical turk for exactly what it was meant for which is making money so we took a hundred dollar bill and divided it into ten thousand teeny pieces and we fed those into the mechanical turk +we asked people to draw what it was that they saw but here there was no sheep like criteria people if they drew a stick figure or a smiley face it actually made it into the bill so what you see is actually a representation of how well people did what it was they were asked to do +so we took these hundred dollar bills and we put them on a website called tenthousandscents com where you can browse through and see all the individual contributions and you can also trade real hundred dollar bills for fake hundred dollar bills and make a donation to the hundred dollar laptop project which is now known as one laptop per child +this is again showing all the different contributions you see some people did beautiful stipple renderings like this one on top spent a long time making realistic versions and other people would draw stick figures or smiley faces here +this is a project called flight patterns what youre looking at is airplane traffic over north america for a twenty four hour period as you see everything starts to fade to black and you see people going to sleep +you may recognize it from two thousand and one a space odyssey when hals dying at the end of the film he starts singing this song as a reference to when computers became human so we resynthesized this song this is what that sounded like we broke down all the individual notes in the singing as well as the phonemes in the singing +and we took all of those individual pieces and we fed them into another turk request this is what it would look like if you went to the site you type in your code +but you first test your mic youd be fed a simple audio clip +after +followed by that you see on the west coast planes moving across the red eye flights to the east coast and youll see everybody waking up on the east coast +followed by european flights coming in the upper right hand corner everybodys moving from the east coast to the west coast you see san francisco and los angeles start to make their journeys down to hawaii in the lower left hand corner i think its one thing to say theres one hundred and forty thousand planes being monitored by the federal government at any one time and its another thing to see that system as it ebbs and flows +and this was seen by a director in l a named james frost who said wait a minute you mean we can shoot a music video without actually using any video +so we did exactly that we made a music video for one of my favorite bands radiohead and i think one of my favorite parts of this project was not just shooting a video with lasers but we also open sourced it and we made it released as a google code project where people could download a bunch of the data and some source code to build their own versions of it and people were making some amazing things this is actually two of my favorites the +so with everybody making so much amazing stuff and actually understanding what it was they were working on i was really interested in trying to make a collaborative project where people were working together to build something and i met a music video director named chris milk and we started bouncing around ideas to make a collaborative music video project but we knew we really needed the right person to kind of rally behind and build something for +so we put the idea on the back burner for a few months and he ended up talking to rick rubin who was finishing up johnny cashs final album +called aint no grave the lyrics to the leading track are aint no grave can hold my body down so we thought this was the perfect project to build a collaborative memorial and a virtual resurrection for johnny cash so i teamed up with my good friend ricardo cabello also known as mr doob whos a much better programmer than i am +and he made this amazing flash drawing tool as you know an animation is +a series of images so what we did was cross cut a bunch of archival footage of johnny cash and at eight frames a second we allowed individuals to draw a single frame that would get woven into this dynamically changing music video +so i dont have time to play the entire thing for you but i want to show you two short clips one is the beginning of the music video and thats going to be followed by a short clip of people who have already contributed to the project talking about it briefly +this is a time lapse image of that exact same data but ive color coded it by type so you can see the diversity of aircraft that are in the skies above us +you can see the person who drew that individual thumbnail and where they were located and if you find one that youre interested in you can actually click on it and open up an information panel where youre able to rate that frame which helps it bubble up to the top +and then this is again the abstract version +which ends up getting a little bit crazy +so the last project i want to talk to you about is another collaboration with chris milk and this is called the wilderness downtown its an online music video for the arcade fire chris and i were really amazed by the potential now with modern web browsers where you have html five audio and video and the power of javascript to render amazingly fast +but most importantly i think +we really wanted to make an experience that was unlike the johnny cash project where you had a small group of people spending a lot of time to contribute something for everyone +what if we had a very low commitment but delivered something individually unique to each person who contributed +so the project starts off by asking you to enter the address of the home where you grew up and you type in the address it actually creates a music video specifically for you pulling in google maps and streetview images into the experience itself so this should really be seen at home with you typing in your own address but im going to give you a little preview of what you can expect +and i remember watching a kid playing on a car stop he was just a toddler and he wasnt very good at it and he kept falling over but i bet playing with this car stop taught him a really valuable lesson and thats that large things dont let you get right past them and that they stay in one place +and so this is a great conceptual model to have of the world +unless youre a particle physicist itd be a terrible model for a particle physicist because they dont play with car stops they play with these little weird particles and when they play with their particles they find they do all sorts of really weird things like they can fly right through walls or they can be in two different places at the same time +and so they wrote down all these observations and they called it the theory of quantum mechanics and so thats where physics was at a few years ago you needed quantum mechanics to describe little tiny particles but you didnt need it to describe the large everyday objects around us +this didnt really sit well with my intuition and maybe its just because i dont play with particles very often well i play with them sometimes but not very often and ive never seen them i mean nobodys ever seen a particle +but it didnt sit well with my logical side either because if everything is made up of little particles and all the little particles follow quantum mechanics then shouldnt everything just follow quantum mechanics +and so id feel a lot better about the whole thing if we could somehow show that an everyday object also follows quantum mechanics so a few years ago i set off to do just that +so i made one this is the first object +that you can see that has been in a mechanical quantum superposition +this device has the ability to be in a quantum superposition but it needs a little help to do it here let me give you an analogy +i dont want to bother them or frankly scare them +so quantum mechanics says that inanimate objects feel the same way the fellow passengers for inanimate objects are not just people but its also the light shining on it and the wind blowing past it and the heat of the room +and so we knew if we wanted to see this piece of metal behave quantum mechanically were going to have to kick out all the other passengers and so thats what we did +instead of just sitting perfectly still it was vibrating and the way it was vibrating was breathing something like this like expanding and contracting bellows and by giving it a gentle nudge we were able to make it both vibrate and not vibrate at the same time +something thats only allowed with quantum mechanics so what im telling you here is something truly +this would be someone whos entirely intuitive +which in turn means the entire chunk of metal is in two different places i think this is really cool +so where would you put your brain on this scale some of us may have opted for one of these extremes but i think for most people in the audience your brain is something like this with a high aptitude in both hemispheres at the same time its not like theyre mutually exclusive or anything you can be logical and intuitive +then why not you +so imagine if youre in multiple places at the same time +how would your consciousness handle your body being delocalized in space +theres one more part to the story its when we warmed it up and we turned on the lights and looked inside the box we saw that the piece metal was still there in one piece +and so i had to develop this new intuition that it seems like all the objects in the elevator are really just quantum objects just crammed into a tiny space you hear a lot of talk about how quantum mechanics says that everything is all interconnected well thats not quite right its more than that +and so i consider myself one of these people along with most of the other experimental quantum physicists who need a good deal of logic to string together these complex ideas but at the same time we need a good deal of intuition to actually make the experiments work +how do we develop this intuition well we like to play with stuff so we go out and play with it and then we see how it acts and then we develop our intuition from there and really you do the same thing so some intuition that you may have developed over the years is that one thing is only in one place at a time +i mean it can sound weird to think about one thing being in two different places at the same time but you werent born with this notion you developed it +truly awesome i knew i had to take a banjo with me to china +and i can tell you that i didnt go to china to become a lawyer in fact i went to nashville +and after a few months i was writing songs and the first song i wrote was in english and the second one was in chinese +and ive played thousands of shows and ive collaborated with so many incredible inspirational musicians around the world and i see the power of music i see the power of music to connect +and asked me what i was going to do with my life i would have told you +cultures i see it when i stand on a stage in a bluegrass festival in east virginia and i look out at the sea of lawn chairs and i bust out into a song in chinese +and everybodys eyes just pop wide open like its going to +and i bust out into a song in chinese and everybody sings along and they roar with delight at this girl with the hair and the instrument and shes singing their music +and i see even more importantly the power of music to connect hearts like the time i was in sichuan province and i was singing for kids in relocation schools in the earthquake disaster zone and this little girl comes up to me +big sister wong washburn wong same difference +big sister wong can i sing you a song that my mom sang for me before she was swallowed in the earthquake +and i sat down she sat on my lap she started singing +and the warmth of her body +was a place i could have stayed forever and in that moment we werent our american selves we werent our chinese selves we were just +mortals +sitting together in that light that keeps us here +ever thought it would have anything to do with the banjo +beautiful the sound of docs voice and the rippling groove of the banjo and after being +totally and completely obsessed with the mammoth richness and history of chinese culture it was like this total relief to hear something so truly american +a +when he saw me on what turned out to be his last hours on this earth his hands moved as if in slow motion and as i wondered what he was up to +his stick fingers made their way up to his pajama shirt fumbling with his buttons +i realized that he was wanting to expose his wicker basket chest to me it was an offering an invitation i did not decline +when we shortcut the physical exam when we lean towards ordering tests instead of talking to and examining the patient we not only overlook simple diagnoses that can be diagnosed at a treatable early stage but were losing much more than that were losing a ritual +no this ritual was about the one message that physicians have needed to convey to their patients although god knows of late in our hubris we seem to have drifted away we seem to have forgotten +as though with the explosion of knowledge the whole human genome mapped out at our feet we are lulled into inattention forgetting that the ritual is cathartic to the physician necessary for the patient forgetting that the ritual has meaning and a singular message to convey to the patient +and the message which i didnt fully understand then even as i delivered it and which i understand better now is this i will always always always be there +i will see you through this i will never abandon you +i will be with you through the end thank you very +were losing a ritual that i believe is transformative transcendent and is at the heart of the patient physician relationship +this may actually be heresy to say this at ted but id like to introduce you to the most important innovation i think in medicine to come in the next ten years and that is the power of the human hand +to touch to comfort to diagnose and to bring about treatment id like to introduce you first to this person whose image you may or may not recognize this is sir arthur conan doyle since were in edinburgh im a big fan of conan doyle you might not know that conan doyle went to medical school here in edinburgh +and his character sherlock holmes was inspired by sir joseph bell joseph bell was an extraordinary teacher by all accounts and conan doyle writing about bell described the following exchange between bell and his students so picture bell sitting in the outpatient department students all around him +patients signing up in the emergency room and being registered and being brought in and a woman comes in with a child and conan doyle describes the following exchange +she says it was good and he says what did you do with the other child she says i left him with my sister at leith +and he says and did you take the shortcut down inverleith row to get here to the infirmary +and bell then goes on to explain to the students he says you see when she said good morning +i picked up her fife accent and the nearest ferry crossing from fife is from burntisland and so she must have taken the ferry over +you notice that the coat shes carrying is too small for the child who is with her and therefore she started out the journey with two children but dropped one off along the way +you notice the clay on the soles of her feet such red clay is not found within a hundred miles of edinburgh except in the botanical gardens and therefore she took a short +and when bell actually strips the patient begins to examine the patient you can only imagine how much more he would discern and as a teacher of medicine as a student myself i was so inspired by that story +within a few minutes she went into cardiac collapse she was resuscitated stabilized whisked over to a cat scan suite right next to the emergency room because they were concerned about blood clots in the lung +his father used to go down into the basement to tap on the sides of casks of wine to determine how much wine was left and whether to reorder +and so when auenbrugger became a physician he began to do the same thing he began to tap on the chests of his patients on their abdomens and basically everything we know about percussion which you can think of as an ultrasound of its day +organ enlargement fluid around the heart fluid in the lungs abdominal changes all of this he described in this wonderful manuscript inventum novum new invention which would have disappeared into obscurity except for the fact that this physician corvisart a famous french physician +famous only because he was physician to this gentleman corvisart repopularized and reintroduced the work and it was followed a year or two later by laennec discovering the stethoscope +that the barber pole the red and white stripes represents the blood bandages of the barber surgeon and the receptacles on either end represent the pots in which the blood was collected +and the cat scan revealed no blood clots in the lung +luke fildes was commissioned to paint this by tate who then established the tate gallery and tate asked fildes to paint a painting of social importance and its interesting that fildes picked this topic fildes oldest son philip died at the age of nine on christmas eve +taken by the physician who held vigil at the bedside for two three nights that he decided that he would try and depict the physician in our time almost a tribute to this physician and hence the painting the doctor a very famous painting its been on calendars postage stamps in many different countries ive often wondered +for where he had the patient ive gotten into some trouble in silicon valley for saying that the patient in the bed has almost become an icon for the real patient whos in the computer +ive actually coined a term for that entity in the computer i call it the ipatient the ipatient is getting wonderful care all across america +the real patient often wonders where is everyone when are they going to come by and explain things to me whos in charge +theres a real disjunction between the patients perception and our own perceptions as physicians of the best medical care i want to show you a picture of what rounds looked like +when i was in training the focus was around the patient we went from bed to bed the attending physician was in charge too often these days rounds look very much like this where the discussion is taking place in a room +far away from the patient the discussion is all about images on the computer data and the one critical piece missing is that of the patient now ive been influenced in this thinking +by two anecdotes that i want to share with you +one had to do with a friend of mine who had a breast cancer +back in our own town getting her subsequent care with her private oncologist and i pressed her and i asked her why did you +come back and get your care +the cancer center was wonderful it had a +beautiful facility giant atrium valet parking a piano that played itself a concierge that took you around from here to there but +she said but they did not touch my breasts +to her it mattered deeply it was enough +for her to make the decision to get her subsequent care +with her private oncologist who every time she went examined both breasts including the axillary tail examined her axilla carefully examined her cervical region her inguinal region did a thorough exam and to her that spoke of a kind of attentiveness that she needed +i was very influenced by that anecdote i was also influenced by another experience that i had again when i was in texas before i moved to stanford i had a reputation as being interested in patients with chronic fatigue this is not a reputation you would wish on your worst enemy +i say that because these are difficult patients they have often been rejected by their families have had bad experiences with medical care and they come to you fully prepared for you to join the long list of people whos about to disappoint them +and i learned very early on with my first patient that i could not do justice to this very complicated patient with all the records they were bringing in a new patient visit of forty five minutes there was just no way +and if i tried id disappoint +we know the average american physician interrupts their patient in fourteen seconds and if i ever get to heaven it will be because i held my piece for forty five minutes and did not interrupt my patient +i then scheduled the physical exam for two weeks hence and when the patient came for the physical +i was able to do a thorough physical because i had nothing else to do i like to think that i do a thorough physical exam but because the whole visit was now about the physical i could do an extraordinarily thorough exam +and i remember my very first patient in that series +and when my ritual began +this very voluble patient began to quiet down +and i remember having a very eerie sense that the patient and i +in which i had a role and the patient had a role +i have never been examined like this before +now if that were true its a true condemnation of our health care system because they had been seen in other places i then proceeded to tell the patient once the patient was dressed the standard things that the person must have heard in other institutions which is this is not in your head this is real +unfortunately it happens all the time +the good news its not cancer its not tuberculosis its not coccidioidomycosis or some obscure fungal infection the bad news is we dont know exactly whats causing this but heres what you should do heres what we should do +i joke but i only half joke that if you come to one of our hospitals missing a limb no one will believe you till they get a cat scan mri or orthopedic consult +and i would lay out all +the standard treatment options that the patient had heard elsewhere +and i always felt that if my patient +gave up the quest for the magic doctor the magic treatment and began with me on a course towards wellness it was because i had earned the right to tell them these things by virtue of the examination something of importance had transpired in the exchange +and they immediately said to me well you are describing a classic ritual +and they helped me understand that rituals are all about transformation +we marry for example with great pomp and ceremony and expense to signal our departure from a life of solitude and misery and loneliness to one of eternal bliss +we signal transitions of power with rituals we signal the passage of a life +with rituals rituals are terribly important theyre all about transformation well i would submit to you +and then incredibly on top of that disrobing and allowing touch +i would submit to you that that is a ritual of exceeding importance and if you shortchange that ritual +by not undressing the patient by listening with your stethoscope on top of the nightgown by not doing a complete exam you have bypassed on the opportunity to seal the patient physician relationship +i am a writer and i want to close by reading you a short passage that i wrote that has to do very much with this scene im an infectious disease physician and in the early days of hiv before we had our medications i presided over so many scenes like this +i am not a luddite i teach at stanford im a physician practicing with cutting edge technology but id like to make the case to you in the next seventeen minutes that +i remember every time i went to a patients deathbed whether in the hospital +or at home i remember my sense of failure +i would look at the tongue i would percuss the chest i would listen to the heart +i would feel the abdomen +i remember so many +patients their names still vivid on my tongue their faces still so clear i remember so many huge hollowed out haunted eyes staring up at me as i performed this ritual and then the next day i would come and i would do it again +and i wanted to read you this one closing passage about one patient +i recall one patient who was at that point no more than a skeleton encased in shrinking skin unable to speak his mouth crusted with candida that was resistant to the usual medications +we had the battle between jefferson and hamilton in one thousand nine hundred and thirteen we had this ugly battle over the federal reserve when it was created with +vicious angry arguments over how it would be constituted and a general agreement that the way it was constituted was the worst possible compromise a compromise guaranteed to destroy this valuable thing this dollar +but then everyone agreeing okay so long as were on the gold standard it should be okay the fed cant mess it up so badly +but then we got off the gold standard for individuals during the depression and we got off the gold standard as a source of international currency coordination during richard nixons presidency each of those times we were on the verge of complete collapse +and nothing happened at all throughout it all the dollar has been one of the most long standing stable reasonable currencies and we all use it every single day no matter what the people screaming about tell us no matter how scared were supposed to be +and this long term fiscal picture that were in right now i think what is most maddening about it is if congress were simply able +to show not that they agree with each other not that theyre able to come up with the best possible compromise but that they are able to just begin the process towards compromise we all instantly are better off the fear +and the longer we put that off the more we make the world nervous the higher interest rates are going to be the quicker were going to +have to face a day of horrible calamity and so just the act of compromise itself and sustained real compromise would give us even more time would allow both sides even longer to spread out the pain and reach even more compromise down the road +so im in the media i feel like my job to make this happen is to help foster the things that seem to lead to compromise to not talk about this in those vague and scary terms that do polarize us but to just talk about it like what it is not an existential crisis not some +to give you a quick primer on where we are a quick refresher on where we are so the fiscal cliff i was told that thats too partisan a thing to say although i cant remember which party +battle between two fundamentally different religious views but a math problem a really solvable math problem one where were not all going to get what we want and one where you know theres going to be a little pain to spread around +its supporting or attacking people say we should call it the fiscal slope or we should call it an austerity crisis but then other people say no thats even more partisan +so i just call it the self imposed self destructive arbitrary deadline about resolving an inevitable problem +the light blue dotted line represents the congressional budget offices best guess of what will happen if congress really doesnt do anything and as you can see sometime around two thousand and twenty seven we reach greek levels of debt somewhere around one hundred and thirty percent of gdp +which tells you that some time in the next twenty years if congress does absolutely nothing were going to hit a moment where the worlds investors the worlds bond buyers are going to say we don +heres another way to look at exactly the same problem +the dark blue line is how much the government spends the light blue line is how much the government gets in and as you can see for most of recent history except for a brief period we have consistently spent more than we take in thus the national debt +and thirty and this graph sort of sums up +what the problem is the democrats they say well this isnt a big deal we can just raise taxes a bit and close that gap especially if we raise taxes on the rich +the republicans say hey no no weve got a better idea why dont we lower both lines why dont we lower government spending and lower government taxes and then well be on an even more favorable long term deficit trajectory +and behind this powerful disagreement between how to close that gap theres the worst kind of cynical party politics the worst kind of insider baseball lobbying all of that stuff but theres also this +powerfully interesting respectful disagreement between two fundamentally different economic philosophies and i like to think +when i picture how republicans see the economy what i picture is +just some amazingly well engineered machine some perfect machine unfortunately i picture it made in germany or japan but this amazing machine that +builds up the more productive areas and lets the less productive areas fade away and die and as a result the whole system is so much more efficient so much richer for everybody +and this view generally believes that there is a role for government a small role to set the rules so people arent lying and cheating and hurting each other maybe you know have a police force and a fire department and an army but to have a very limited reach into the mechanisms of this machinery +and when i picture how democrats and democratic leaning economists picture this economy most democratic economists are you know theyre capitalists they believe yes thats a good system a lot of the time its good to let markets move resources to their more productive use but that system has tons of problems +angry negotiations negotiations breaking apart reports of phone calls that arent going well people saying nothings happening at all and then sometime around christmas or new years were going to hear okay they resolved everything +that make this life worse for all of us and so the government does have a role +to take resources from more productive uses or from richer sources and give them to other sources +and when you think about the economy through these two different lenses +you understand why this crisis is so hard to solve because the worse the crisis gets the higher the stakes are the more each side thinks they know the answer and the other side is just going to ruin everything +and i can get really despairing ive spent a lot of the last few years really depressed about this until this year i learned something that i felt really excited about i feel like its really good news and its so shocking i dont like saying it because i think +people wont believe me but heres what i learned the american people taken as a whole when it comes to these issues to fiscal issues are moderate pragmatic centrists and i know thats hard to believe that the american people are moderate pragmatic centrists but let me explain what im thinking +when you look at how the federal government spends money so this is the battle right here +fifty five percent more than half is on social security medicare medicaid a few other health programs twenty percent defense nineteen percent discretionary and six percent interest so +when were talking about cutting government spending this is the pie were talking about and americans overwhelmingly and it doesnt matter what party theyre in overwhelmingly like that big fifty five percent chunk +they like social security they like medicare they even like medicaid even though that goes to the poor and indigent which you might think would have less support +and they do not want it fundamentally touched although the american people are remarkably comfortable +and democrats roughly equal to republicans with some minor tweaks to make the system more stable social security is fairly easy to fix the rumors of its demise are always greatly exaggerated so gradually raise social security retirement age maybe only on people not yet born +he told me that a few months ago he said hes ninety eight percent positive theyre going to resolve it +americans are about fifty fifty whether theyre democrats or republicans reduce medicare for very wealthy seniors seniors who make a lot of money dont even eliminate it just reduce it people generally are +we are not a nation thats powerfully divided on the major major issue were comfortable with it needing some tweaks but we want to keep it were not open to a discussion of eliminating it +and i got an email from him today saying all right were basically on track but now im eighty percent positive that theyre going to resolve it +now there is one issue that is hyper partisan and where there is one party that is just spend spend spend we dont care spend some more and that of course is republicans when it comes to military defense spending they way outweigh democrats the vast majority want to protect +military defense spending thats twenty percent of the budget +and that presents a more difficult issue +i should also note that the discretionary spending which is about nineteen percent of the budget that is democratic and republican issues so you do have welfare food stamps other programs that tend to be popular among democrats but you also have the farm bill and all sorts of department of interior inducements for +oil drilling and other things which tend to be popular among republicans +now when it comes to taxes there is more disagreement thats a more partisan area you have democrats overwhelmingly supportive of raising the income tax on people who make two hundred and fifty thousand dollars a year republicans +sort of against it although if you break it out by income republicans who make less than seventy five thousand dollars a year like this idea so basically republicans who make more than two hundred and fifty thousand dollars a year dont want to be taxed +raising taxes on investment income you also see about two thirds of democrats but only one third of republicans are comfortable with that idea +and it made me think i love studying these moments in american history when there was this frenzy of partisan anger that the economy was on the verge of total collapse +this brings up a really important point which is that we tend in this country to talk about democrats and republicans and think theres this little group over there called independents thats what two percent +if you add democrats you add republicans youve got the american people +but that is not the case at all and it has not been the case for most of modern american history roughly a third of americans say that they are democrats around a quarter say that they are republicans a tiny little sliver +call themselves libertarians or socialists or some other small third party +and the largest block forty percent say theyre independents so most americans are not partisan and most of the people in the independent camp fall somewhere in between so even though we have tremendous overlap between the views on these fiscal issues of democrats and republicans +we have even more overlap when you add in the independents +now we get to fight about all sorts of other issues we get to hate each other on gun control and abortion and the environment +but on these fiscal issues these important fiscal issues we just are not anywhere nearly as divided as people say and in fact theres this other group of people who are not as divided as people might think and that group is economists +i talk to a lot of economists and +you were a free market capitalist economist or you were a keynesian liberal economist and these people didnt go to each others weddings +the most famous early battle was alexander hamilton and thomas jefferson over what the dollar would be and how it would be backed up with alexander hamilton saying we need a central bank the first bank of the united states or else the dollar will have no value this economy wont work and +but in my experience it is really really hard to find an economist under forty who still has that kind of way of seeing the world the vast majority of economists it is so uncool to call yourself an ideologue of either camp +the phrase that you want if youre a graduate student or a postdoc or youre a professor a thirty eight year old economics professor is im an empiricist i go by the data +and the data is very clear none of these major theories have been completely successful the twentieth century the last hundred years is riddled with disastrous examples of times that one school or the other tried to explain the past or predict the future and just did an awful awful job so +the economics profession has acquired some degree of modesty +they still are an awfully arrogant group of people i will assure you but theyre now arrogant about their impartiality and they too see a tremendous range of potential outcomes +and this nonpartisanship is something that exists that has existed in secret in america for years and years and years ive spent a lot of the fall +talking to the three major organizations that survey american political attitudes pew research the university of chicagos national opinion research center and +support no we mustnt tax and we must limit the size of government or no we must encourage government to play a larger role in redistribution and correcting the ills of capitalism those groups are very very small the vast majority of people +they pick and choose they see compromise and they change over time when they hear a better argument or a worse argument +and that part of it has not changed what has changed is how people respond to vague questions if you ask people vague questions like +do you think there should be more government or less government do you think government +should especially if you use loaded language do you think the government should provide handouts or do you think the government should redistribute then you can see radical partisan change but when you get specific when you actually ask about the actual taxing and spending issues under consideration +thomas jefferson saying the people wont trust that they just fought off a king theyre not going to accept some central authority this battle defined the first one hundred and fifty years of the u s economy and at every moment different partisans saying oh my god the economys about to collapse +people are remarkably centrist theyre remarkably open to compromise +so what we have then when you think about the fiscal cliff dont think of it as +the american people fundamentally cant stand each other on these issues and that we must be ripped apart into two separate warring nations +think of it as a tiny tiny number of ancient economists +and misrepresentative ideologues have captured the process and theyve captured the process through familiar ways through a primary system which encourages that small group of peoples voices +because that small group of people the people who answer all yeses or all noes on those ideological questions they might be small but every one of them has a blog every one of them has been on fox or msnbc in the last week +every one of them becomes a louder and louder voice but they dont represent us they dont represent what our views are +and that gets me back to the dollar and it gets me back to reminding myself that we know this experience we know what its like +to have these people on tv in congress yelling about how the end of the world is coming if we dont adopt their view completely because its happened about the dollar ever since theres been a dollar +rock a mustache not a beard not a goatee a mustache for the thirty days of november and then we agreed that we would come together at the end of the month have a mustache themed party +and award a prize for the best and of course the worst mustache +think the beautiful malin akerman put it perfectly every man deserves the opportunity to grow a little bit of luxury +hipster mustache +it created a lot of controversy +hated it parents would shuffle kids away from +we came together at the end of the month and we celebrated our journey and it was a real journey and we had a lot of fun and in two thousand and four i said to the guys that was so much fun we need to legitimize this so we can get away with it year on year +so we married growing a mustache with prostate cancer and then we created our tagline which is changing the face of mens health and that eloquently describes the challenge changing your appearance for the thirty days and also the outcome that were trying to achieve getting men engaged in their health +the ceo of the prostate cancer foundation +i said to him ive got the most amazing idea thats going to transform your organization +and funds for his organization +and i said were going to come together at the end were going to have a mustache themed party were going to have djs were going to celebrate life and were going to change the face of mens health and he just looked at me and laughed and he said +but were an ultraconservative organization we cant have anything to do with you diff --git a/kenlm b/kenlm new file mode 160000 index 00000000..689a25aa --- /dev/null +++ b/kenlm @@ -0,0 +1 @@ +Subproject commit 689a25aae9171b3ea46bd80d4189f540f35f1a02 diff --git a/native_client/BUILD b/native_client/BUILD index 5785b9a3..9119f392 100644 --- a/native_client/BUILD +++ b/native_client/BUILD @@ -225,11 +225,16 @@ cc_binary( "@com_google_absl//absl/types:optional", "@boost//:program_options", ], + linkstatic = 1, linkopts = [ "-lm", "-ldl", "-pthread", - ], + ] + select({ + # ARMv7: error: Android 5.0 and later only support position-independent executables (-fPIE). + "//tensorflow:android": ["-fPIE -pie"], + "//conditions:default": [], + }), ) cc_binary( diff --git a/taskcluster/.shared.yml b/taskcluster/.shared.yml index a5e88ab1..718b0d83 100644 --- a/taskcluster/.shared.yml +++ b/taskcluster/.shared.yml @@ -11,6 +11,13 @@ training: deepspeech: packages_xenial: apt: 'make build-essential gfortran git libblas-dev liblapack-dev libsox-dev libmagic-dev libgsm1-dev libltdl-dev libpng-dev python python-dev zlib1g-dev' +kenlm: + packages_xenial: + apt: 'apt-get -qq update && apt-get -qq -y install cmake realpath build-essential libboost-dev wget software-properties-common zlib1g-dev libbz2-dev liblzma-dev libboost-program-options-dev libboost-system-dev libboost-thread-dev libboost-test-dev' + packages_android: + apt: 'apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get -qq -y install cmake coreutils build-essential wget software-properties-common unzip' + packages_win: + pacman: 'pacman --noconfirm -S cmake mingw-w64-x86_64-boost mingw-w64-x86_64-toolchain zlib bzip2 xz' tensorflow: packages_xenial: apt: 'apt-get -qq update && apt-get -qq -y install realpath build-essential python-virtualenv python-dev python-pip libblas-dev liblapack-dev gfortran wget software-properties-common pixz zip zlib1g-dev unzip' @@ -125,6 +132,31 @@ system: win: url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.pyenv.win.8/artifacts/public/pyenv.tar.gz' namespace: 'project.deepspeech.pyenv.win.8' + kenlm: + android_arm64_cpu: + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.kenlm.689a25aae9171b3ea46bd80d4189f540f35f1a02.4.android-arm64/artifacts/public/kenlm.tar.gz" + namespace: "project.deepspeech.kenlm.689a25aae9171b3ea46bd80d4189f540f35f1a02.4.android-arm64" + android_armv7_cpu: + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.kenlm.689a25aae9171b3ea46bd80d4189f540f35f1a02.4.android-armv7/artifacts/public/kenlm.tar.gz" + namespace: "project.deepspeech.kenlm.689a25aae9171b3ea46bd80d4189f540f35f1a02.4.android-armv7" + android_x86_64_cpu: + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.kenlm.689a25aae9171b3ea46bd80d4189f540f35f1a02.4.android-x86_64/artifacts/public/kenlm.tar.gz" + namespace: "project.deepspeech.kenlm.689a25aae9171b3ea46bd80d4189f540f35f1a02.4.android-x86_64" + linux_amd64_cpu: + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.kenlm.689a25aae9171b3ea46bd80d4189f540f35f1a02.4.cpu/artifacts/public/kenlm.tar.gz" + namespace: "project.deepspeech.kenlm.689a25aae9171b3ea46bd80d4189f540f35f1a02.4.cpu" + linux_arm64_cpu: + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.kenlm.689a25aae9171b3ea46bd80d4189f540f35f1a02.4.arm64/artifacts/public/kenlm.tar.gz" + namespace: "project.deepspeech.kenlm.689a25aae9171b3ea46bd80d4189f540f35f1a02.4.arm64" + linux_rpi3_cpu: + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.kenlm.689a25aae9171b3ea46bd80d4189f540f35f1a02.4.rpi3/artifacts/public/kenlm.tar.gz" + namespace: "project.deepspeech.kenlm.689a25aae9171b3ea46bd80d4189f540f35f1a02.4.rpi3" + darwin_amd64_cpu: + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.kenlm.689a25aae9171b3ea46bd80d4189f540f35f1a02.4.osx/artifacts/public/kenlm.tar.gz" + namespace: "project.deepspeech.kenlm.689a25aae9171b3ea46bd80d4189f540f35f1a02.4.osx" + win_amd64_cpu: + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.kenlm.689a25aae9171b3ea46bd80d4189f540f35f1a02.4.win/artifacts/public/kenlm.tar.gz" + namespace: "project.deepspeech.kenlm.689a25aae9171b3ea46bd80d4189f540f35f1a02.4.win" swig: repo: "https://github.com/lissyx/swig" sha1: "1a4c14945012f1282c2eddc174fb7674d5295de8" diff --git a/taskcluster/kenlm_android-arm64-cpu-opt.yml b/taskcluster/kenlm_android-arm64-cpu-opt.yml new file mode 100644 index 00000000..934ee955 --- /dev/null +++ b/taskcluster/kenlm_android-arm64-cpu-opt.yml @@ -0,0 +1,17 @@ +build: + template_file: generic_tc_caching-linux-opt-base.tyml + cache: + artifact_url: ${system.kenlm.android_arm64_cpu.url} + artifact_namespace: ${system.kenlm.android_arm64_cpu.namespace} + system_config: + > + ${kenlm.packages_android.apt} + docker_image: "ubuntu:20.04" + scripts: + setup: "taskcluster/kenlm_tc-setup.sh --android-arm64" + build: "taskcluster/kenlm_tc-build.sh --android-arm64" + package: "taskcluster/kenlm_tc-package.sh" + workerType: "${docker.dsBuild}" + metadata: + name: "KenLM Android ARM64 CPU" + description: "Building KenLM for Android/ARM64, CPU only, optimized version" diff --git a/taskcluster/kenlm_android-armv7-cpu-opt.yml b/taskcluster/kenlm_android-armv7-cpu-opt.yml new file mode 100644 index 00000000..3dbc4de6 --- /dev/null +++ b/taskcluster/kenlm_android-armv7-cpu-opt.yml @@ -0,0 +1,17 @@ +build: + template_file: generic_tc_caching-linux-opt-base.tyml + cache: + artifact_url: ${system.kenlm.android_armv7_cpu.url} + artifact_namespace: ${system.kenlm.android_armv7_cpu.namespace} + system_config: + > + ${kenlm.packages_android.apt} + docker_image: "ubuntu:20.04" + scripts: + setup: "taskcluster/kenlm_tc-setup.sh --android-armv7" + build: "taskcluster/kenlm_tc-build.sh --android-armv7" + package: "taskcluster/kenlm_tc-package.sh" + workerType: "${docker.dsBuild}" + metadata: + name: "KenLM Android ARMv7 CPU" + description: "Building KenLM for Android/ARMv7, CPU only, optimized version" diff --git a/taskcluster/kenlm_android-x86_64-cpu-opt.yml b/taskcluster/kenlm_android-x86_64-cpu-opt.yml new file mode 100644 index 00000000..346b7ed5 --- /dev/null +++ b/taskcluster/kenlm_android-x86_64-cpu-opt.yml @@ -0,0 +1,17 @@ +build: + template_file: generic_tc_caching-linux-opt-base.tyml + cache: + artifact_url: ${system.kenlm.android_x86_64_cpu.url} + artifact_namespace: ${system.kenlm.android_x86_64_cpu.namespace} + system_config: + > + ${kenlm.packages_android.apt} + docker_image: "ubuntu:20.04" + scripts: + setup: "taskcluster/kenlm_tc-setup.sh --android-x86_64" + build: "taskcluster/kenlm_tc-build.sh --android-x86_64" + package: "taskcluster/kenlm_tc-package.sh" + workerType: "${docker.dsBuild}" + metadata: + name: "KenLM Android x86_64 CPU" + description: "Building KenLM for Android/x86_64, CPU only, optimized version" diff --git a/taskcluster/kenlm_darwin-amd64-cpu-opt.yml b/taskcluster/kenlm_darwin-amd64-cpu-opt.yml new file mode 100644 index 00000000..60567b4d --- /dev/null +++ b/taskcluster/kenlm_darwin-amd64-cpu-opt.yml @@ -0,0 +1,13 @@ +build: + template_file: generic_tc_caching-darwin-opt-base.tyml + cache: + artifact_url: ${system.kenlm.darwin_amd64_cpu.url} + artifact_namespace: ${system.kenlm.darwin_amd64_cpu.namespace} + scripts: + setup: "taskcluster/kenlm_tc-setup.sh --macos-amd64" + build: "taskcluster/kenlm_tc-build.sh --macos-amd64" + package: "taskcluster/kenlm_tc-package.sh" + workerType: ${macOS.dsBuild} + metadata: + name: "KenLM macOS AMD64 CPU" + description: "Building KenLM for macOS/AMD64, CPU only, optimized version" diff --git a/taskcluster/kenlm_linux-amd64-cpu-opt.yml b/taskcluster/kenlm_linux-amd64-cpu-opt.yml new file mode 100644 index 00000000..e11a1a28 --- /dev/null +++ b/taskcluster/kenlm_linux-amd64-cpu-opt.yml @@ -0,0 +1,16 @@ +build: + template_file: generic_tc_caching-linux-opt-base.tyml + cache: + artifact_url: ${system.kenlm.linux_amd64_cpu.url} + artifact_namespace: ${system.kenlm.linux_amd64_cpu.namespace} + system_config: + > + ${kenlm.packages_xenial.apt} + scripts: + setup: "taskcluster/kenlm_tc-setup.sh --linux-amd64" + build: "taskcluster/kenlm_tc-build.sh --linux-amd64" + package: "taskcluster/kenlm_tc-package.sh" + workerType: "${docker.dsBuild}" + metadata: + name: "KenLM Linux AMD64 CPU" + description: "Building KenLM for Linux/AMD64, CPU only, optimized version" diff --git a/taskcluster/kenlm_linux-arm64-cpu-opt.yml b/taskcluster/kenlm_linux-arm64-cpu-opt.yml new file mode 100644 index 00000000..018579d0 --- /dev/null +++ b/taskcluster/kenlm_linux-arm64-cpu-opt.yml @@ -0,0 +1,16 @@ +build: + template_file: generic_tc_caching-linux-opt-base.tyml + cache: + artifact_url: ${system.kenlm.linux_arm64_cpu.url} + artifact_namespace: ${system.kenlm.linux_arm64_cpu.namespace} + system_setup: + > + apt-get -qq update && apt-get -qq -y install cmake wget pixz bzip2 multistrap + scripts: + setup: "taskcluster/kenlm_tc-setup.sh --linux-arm64" + build: "taskcluster/kenlm_tc-build.sh --linux-arm64" + package: "taskcluster/kenlm_tc-package.sh" + workerType: "${docker.dsBuild}" + metadata: + name: "KenLM Linux ARM64 CPU" + description: "Building KenLM for Linux/ARM64, CPU only, optimized version" diff --git a/taskcluster/kenlm_linux-rpi3-cpu-opt.yml b/taskcluster/kenlm_linux-rpi3-cpu-opt.yml new file mode 100644 index 00000000..92a43953 --- /dev/null +++ b/taskcluster/kenlm_linux-rpi3-cpu-opt.yml @@ -0,0 +1,16 @@ +build: + template_file: generic_tc_caching-linux-opt-base.tyml + cache: + artifact_url: ${system.kenlm.linux_rpi3_cpu.url} + artifact_namespace: ${system.kenlm.linux_rpi3_cpu.namespace} + system_setup: + > + apt-get -qq update && apt-get -qq -y install cmake wget pixz bzip2 multistrap + scripts: + setup: "taskcluster/kenlm_tc-setup.sh --linux-rpi3" + build: "taskcluster/kenlm_tc-build.sh --linux-rpi3" + package: "taskcluster/kenlm_tc-package.sh" + workerType: "${docker.dsBuild}" + metadata: + name: "KenLM Linux RPi3 CPU" + description: "Building KenLM for Linux/RPi3, CPU only, optimized version" diff --git a/taskcluster/kenlm_multistrap_arm64_buster.conf b/taskcluster/kenlm_multistrap_arm64_buster.conf new file mode 100644 index 00000000..1ccf25c1 --- /dev/null +++ b/taskcluster/kenlm_multistrap_arm64_buster.conf @@ -0,0 +1,14 @@ +[General] +arch=arm64 +noauth=true +unpack=true +debootstrap=Debian +aptsources=Debian +cleanup=true + +[Debian] +packages=libc6 libc6-dev libstdc++-7-dev linux-libc-dev libboost-dev zlib1g-dev libbz2-dev liblzma-dev libboost-program-options-dev libboost-system-dev libboost-thread-dev libboost-test-dev +source=http://deb.debian.org/debian +keyring=debian-archive-keyring +components=main +suite=buster diff --git a/taskcluster/kenlm_multistrap_rpi3_buster.conf b/taskcluster/kenlm_multistrap_rpi3_buster.conf new file mode 100644 index 00000000..0b13f624 --- /dev/null +++ b/taskcluster/kenlm_multistrap_rpi3_buster.conf @@ -0,0 +1,14 @@ +[General] +arch=armhf +noauth=true +unpack=true +debootstrap=Raspbian +aptsources=Raspbian +cleanup=true + +[Raspbian] +packages=libc6 libc6-dev libffi-dev libstdc++-6-dev linux-libc-dev libboost-dev zlib1g-dev libbz2-dev liblzma-dev libboost-program-options-dev libboost-system-dev libboost-thread-dev libboost-test-dev +source=http://raspbian.raspberrypi.org/raspbian/ +keyring=raspbian-archive-keyring +components=main +suite=buster diff --git a/taskcluster/kenlm_tc-build.sh b/taskcluster/kenlm_tc-build.sh new file mode 100755 index 00000000..91f12a08 --- /dev/null +++ b/taskcluster/kenlm_tc-build.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +set -xe + +target=$1 + +source $(dirname $0)/tc-all-vars.sh + +MAKE_TARGETS="lmplz filter build_binary" +CMAKE_BUILD="cmake --build . -j ${DS_CPU_COUNT} --target ${MAKE_TARGETS}" +CMAKE_DEFINES="-DFORCE_STATIC=ON" +case "${target}" in + --android-arm64) + export Boost_DIR="${DS_ROOT_TASK}/DeepSpeech/ds/ndk_21_boost_1.72.0/libs/arm64-v8a/cmake/Boost-1.72.0/" + CMAKE_DEFINES="$CMAKE_DEFINES -DCMAKE_SYSTEM_NAME=Android" + CMAKE_DEFINES="$CMAKE_DEFINES -DCMAKE_SYSTEM_VERSION=21" + CMAKE_DEFINES="$CMAKE_DEFINES -DCMAKE_ANDROID_ARCH_ABI=arm64-v8a" + CMAKE_DEFINES="$CMAKE_DEFINES -DCMAKE_ANDROID_NDK=${ANDROID_NDK_HOME}" + CMAKE_DEFINES="$CMAKE_DEFINES -DCMAKE_ANDROID_STL_TYPE=c++_static" + CMAKE_DEFINES="$CMAKE_DEFINES -DTHREADS_PTHREAD_ARG=2" + ;; + + --android-armv7) + export Boost_DIR="${DS_ROOT_TASK}/DeepSpeech/ds/ndk_21_boost_1.72.0/libs/armeabi-v7a/cmake/Boost-1.72.0/" + CMAKE_DEFINES="$CMAKE_DEFINES -DCMAKE_SYSTEM_NAME=Android" + CMAKE_DEFINES="$CMAKE_DEFINES -DCMAKE_SYSTEM_VERSION=21" + CMAKE_DEFINES="$CMAKE_DEFINES -DCMAKE_ANDROID_ARCH_ABI=armeabi-v7a" + CMAKE_DEFINES="$CMAKE_DEFINES -DCMAKE_ANDROID_NDK=${ANDROID_NDK_HOME}" + CMAKE_DEFINES="$CMAKE_DEFINES -DCMAKE_ANDROID_STL_TYPE=c++_static" + CMAKE_DEFINES="$CMAKE_DEFINES -DTHREADS_PTHREAD_ARG=2" + ;; + + --android-x86_64) + export Boost_DIR="${DS_ROOT_TASK}/DeepSpeech/ds/ndk_21_boost_1.72.0/libs/x86_64/cmake/Boost-1.72.0/" + CMAKE_DEFINES="$CMAKE_DEFINES -DCMAKE_SYSTEM_NAME=Android" + CMAKE_DEFINES="$CMAKE_DEFINES -DCMAKE_SYSTEM_VERSION=21" + CMAKE_DEFINES="$CMAKE_DEFINES -DCMAKE_ANDROID_ARCH_ABI=x86_64" + CMAKE_DEFINES="$CMAKE_DEFINES -DCMAKE_ANDROID_NDK=${ANDROID_NDK_HOME}" + CMAKE_DEFINES="$CMAKE_DEFINES -DCMAKE_ANDROID_STL_TYPE=c++_static" + CMAKE_DEFINES="$CMAKE_DEFINES -DTHREADS_PTHREAD_ARG=2" + ;; + + --macos-amd64) + export KENLM_BREW="${TASKCLUSTER_ORIG_TASKDIR}/homebrew-kenlm" + export PATH=${KENLM_BREW}/bin:$PATH + CMAKE_DEFINES="$CMAKE_DEFINES -DZLIB_LIBRARY=${KENLM_BREW}/opt/zlib/lib/libz.a" + #CMAKE_DEFINES="$CMAKE_DEFINES -DBZIP2_LIBRARIES=${KENLM_BREW}/opt/bzip2/lib/libz2.a" + CMAKE_DEFINES="$CMAKE_DEFINES -DTHREADS_PTHREAD_ARG=2" + ;; + + --linux-arm64) + TOOLCHAIN=${DS_ROOT_TASK}/DeepSpeech/ds/gcc-linaro-7.2.1-2017.11-x86_64_aarch64-linux-gnu/bin/aarch64-linux-gnu + CMAKE_DEFINES="$CMAKE_DEFINES -DCMAKE_SYSTEM_NAME=Linux" + CMAKE_DEFINES="$CMAKE_DEFINES -DCMAKE_SYSTEM_PROCESSOR=aarch64" + CMAKE_DEFINES="$CMAKE_DEFINES -DCMAKE_C_COMPILER=${TOOLCHAIN}-gcc" + CMAKE_DEFINES="$CMAKE_DEFINES -DCMAKE_CXX_COMPILER=${TOOLCHAIN}-g++" + CMAKE_DEFINES="$CMAKE_DEFINES -DCMAKE_SYSROOT=${DS_ROOT_TASK}/DeepSpeech/ds/multistrap-armbian64-buster/" + CMAKE_DEFINES="$CMAKE_DEFINES -DTHREADS_PTHREAD_ARG=2" + CMAKE_BUILD="make -j ${DS_CPU_COUNT} ${MAKE_TARGETS}" + ;; + + --linux-rpi3) + TOOLCHAIN=${DS_ROOT_TASK}/DeepSpeech/ds/gcc-linaro-7.2.1-2017.11-x86_64_arm-linux-gnueabihf/bin/arm-linux-gnueabihf + CMAKE_DEFINES="$CMAKE_DEFINES -DCMAKE_SYSTEM_NAME=Linux" + CMAKE_DEFINES="$CMAKE_DEFINES -DCMAKE_SYSTEM_PROCESSOR=arm" + CMAKE_DEFINES="$CMAKE_DEFINES -DCMAKE_C_COMPILER=${TOOLCHAIN}-gcc" + CMAKE_DEFINES="$CMAKE_DEFINES -DCMAKE_CXX_COMPILER=${TOOLCHAIN}-g++" + CMAKE_DEFINES="$CMAKE_DEFINES -DCMAKE_SYSROOT=${DS_ROOT_TASK}/DeepSpeech/ds/multistrap-raspbian-buster/" + CMAKE_DEFINES="$CMAKE_DEFINES -DTHREADS_PTHREAD_ARG=2" + CMAKE_BUILD="make -j ${DS_CPU_COUNT} ${MAKE_TARGETS}" + ;; + + --linux-amd64) + CMAKE_BUILD="make -j ${DS_CPU_COUNT} ${MAKE_TARGETS}" + ;; + + --windows-amd64) + export Boost_DIR="$TASKCLUSTER_TASK_DIR/boost_1_72_0/lib64-msvc-14.2/cmake/Boost-1.72.0/" + export CMAKE_GENERATOR='Visual Studio 16 2019' + export PATH=$TASKCLUSTER_TASK_DIR/cmake-3.18.2-win64-x64/bin/:$PATH + CMAKE_DEFINES="$CMAKE_DEFINES -A x64" + CMAKE_DEFINES="$CMAKE_DEFINES -DLIBLZMA_LIBRARY=$TASKCLUSTER_TASK_DIR/xz-5.2.5/bin_x86-64/liblzma.a -DLIBLZMA_INCLUDE_DIR=$TASKCLUSTER_TASK_DIR/xz-5.2.5/include/" + #CMAKE_DEFINES="$CMAKE_DEFINES -DBZIP2_LIBRARIES=$TASKCLUSTER_TASK_DIR/bzip2-dev-1.0.8.0-win-x64/libbz2-static.lib -DBZIP2_INCLUDE_DIR=$TASKCLUSTER_TASK_DIR/bzip2-dev-1.0.8.0-win-x64/" + ;; +esac + +mkdir ${DS_ROOT_TASK}/DeepSpeech/ds/kenlm/build/ + +pushd ${DS_ROOT_TASK}/DeepSpeech/ds/kenlm/build/ + export EIGEN3_ROOT=${DS_ROOT_TASK}/DeepSpeech/ds/eigen-3.3.7 + cmake ${CMAKE_DEFINES} ../ + ${CMAKE_BUILD} +popd diff --git a/taskcluster/kenlm_tc-package.sh b/taskcluster/kenlm_tc-package.sh new file mode 100755 index 00000000..93a36050 --- /dev/null +++ b/taskcluster/kenlm_tc-package.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -xe + +source $(dirname $0)/tc-all-vars.sh + +mkdir -p ${TASKCLUSTER_ARTIFACTS} || true + +cd ${DS_ROOT_TASK}/DeepSpeech/ds/kenlm/build/bin/ && \ + tar \ + -czf ${TASKCLUSTER_ARTIFACTS}/kenlm.tar.gz \ + build_binary${PLATFORM_EXE_SUFFIX} \ + filter${PLATFORM_EXE_SUFFIX} \ + lmplz${PLATFORM_EXE_SUFFIX} diff --git a/taskcluster/kenlm_tc-setup.sh b/taskcluster/kenlm_tc-setup.sh new file mode 100755 index 00000000..a778f184 --- /dev/null +++ b/taskcluster/kenlm_tc-setup.sh @@ -0,0 +1,109 @@ +#!/bin/bash + +set -ex + +target=$1 + +source $(dirname $0)/tc-all-vars.sh + +mkdir -p ${TASKCLUSTER_ARTIFACTS} || true + +pushd ${DS_ROOT_TASK}/DeepSpeech/ds/ + case "${target}" in + --android-arm64|--android-armv7|--android-x86_64) + source $(dirname $0)/tc-android-utils.sh + + wget -q https://github.com/dec1/Boost-for-Android/releases/download/ndk_21_boost_1.72.0/ndk_21_boost_1.72.0.zip && unzip ndk_21_boost_1.72.0.zip + mkdir -p ${ANDROID_NDK_HOME} || true + android_install_ndk + ;; + + --macos-amd64) + source $(dirname $0)/homebrew-build.sh + export KENLM_BREW="${TASKCLUSTER_ORIG_TASKDIR}/homebrew-kenlm" + do_prepare_homebrew "${KENLM_BREW}" + + install_pkg_homebrew "cmake" + install_pkg_homebrew "coreutils" + install_pkg_homebrew "boost" + install_pkg_homebrew "bzip2" + install_pkg_homebrew "zlib" + install_pkg_homebrew "xz" + ;; + + --linux-arm64) + wget -q -O - https://releases.linaro.org/components/toolchain/binaries/7.2-2017.11/aarch64-linux-gnu/gcc-linaro-7.2.1-2017.11-x86_64_aarch64-linux-gnu.tar.xz | pixz -d | tar -xf - + multistrap -d multistrap-armbian64-buster/ -f taskcluster/kenlm_multistrap_arm64_buster.conf + + # fix all symlink + ls -hal multistrap-armbian64-buster/usr/lib/aarch64-linux-gnu/libpthread.so + ls -hal multistrap-armbian64-buster/usr/lib/aarch64-linux-gnu/libz.so + ls -hal multistrap-armbian64-buster/usr/lib/aarch64-linux-gnu/libbz2.so + ls -hal multistrap-armbian64-buster/usr/lib/aarch64-linux-gnu/libm.so + + rm multistrap-armbian64-buster/usr/lib/aarch64-linux-gnu/libpthread.so + rm multistrap-armbian64-buster/usr/lib/aarch64-linux-gnu/libz.so + rm multistrap-armbian64-buster/usr/lib/aarch64-linux-gnu/libbz2.so + rm multistrap-armbian64-buster/usr/lib/aarch64-linux-gnu/libm.so + + ln -s ../../../lib/aarch64-linux-gnu/libpthread.so.0 multistrap-armbian64-buster/usr/lib/aarch64-linux-gnu/libpthread.so + ln -s ../../../lib/aarch64-linux-gnu/libz.so.1.2.11 multistrap-armbian64-buster/usr/lib/aarch64-linux-gnu/libz.so + ln -s ../../../lib/aarch64-linux-gnu/libbz2.so.1.0 multistrap-armbian64-buster/usr/lib/aarch64-linux-gnu/libbz2.so + ln -s ../../../lib/aarch64-linux-gnu/libm.so.6 multistrap-armbian64-buster/usr/lib/aarch64-linux-gnu/libm.so + + ls -hal multistrap-armbian64-buster/usr/lib/aarch64-linux-gnu/libpthread.so + ls -hal multistrap-armbian64-buster/usr/lib/aarch64-linux-gnu/libz.so + ls -hal multistrap-armbian64-buster/usr/lib/aarch64-linux-gnu/libbz2.so + ls -hal multistrap-armbian64-buster/usr/lib/aarch64-linux-gnu/libm.so + ;; + + --linux-rpi3) + wget -q -O - https://releases.linaro.org/components/toolchain/binaries/7.2-2017.11/arm-linux-gnueabihf/gcc-linaro-7.2.1-2017.11-x86_64_arm-linux-gnueabihf.tar.xz | pixz -d | tar -xf - + multistrap -d multistrap-raspbian-buster/ -f taskcluster/kenlm_multistrap_rpi3_buster.conf + + # fix all symlink + ls -hal multistrap-raspbian-buster/usr/lib/arm-linux-gnueabihf/libpthread.so + ls -hal multistrap-raspbian-buster/usr/lib/arm-linux-gnueabihf/libz.so + ls -hal multistrap-raspbian-buster/usr/lib/arm-linux-gnueabihf/libbz2.so + ls -hal multistrap-raspbian-buster/usr/lib/arm-linux-gnueabihf/libm.so + + rm multistrap-raspbian-buster/usr/lib/arm-linux-gnueabihf/libpthread.so + rm multistrap-raspbian-buster/usr/lib/arm-linux-gnueabihf/libz.so + rm multistrap-raspbian-buster/usr/lib/arm-linux-gnueabihf/libbz2.so + rm multistrap-raspbian-buster/usr/lib/arm-linux-gnueabihf/libm.so + + ln -s ../../../lib/arm-linux-gnueabihf/libpthread.so.0 multistrap-raspbian-buster/usr/lib/arm-linux-gnueabihf/libpthread.so + ln -s ../../../lib/arm-linux-gnueabihf/libz.so.1.2.11 multistrap-raspbian-buster/usr/lib/arm-linux-gnueabihf/libz.so + ln -s ../../../lib/arm-linux-gnueabihf/libbz2.so.1.0 multistrap-raspbian-buster/usr/lib/arm-linux-gnueabihf/libbz2.so + ln -s ../../../lib/arm-linux-gnueabihf/libm.so.6 multistrap-raspbian-buster/usr/lib/arm-linux-gnueabihf/libm.so + + ls -hal multistrap-raspbian-buster/usr/lib/arm-linux-gnueabihf/libpthread.so + ls -hal multistrap-raspbian-buster/usr/lib/arm-linux-gnueabihf/libz.so + ls -hal multistrap-raspbian-buster/usr/lib/arm-linux-gnueabihf/libbz2.so + ls -hal multistrap-raspbian-buster/usr/lib/arm-linux-gnueabihf/libm.so + ;; + + --windows-amd64) + wget -q https://github.com/Kitware/CMake/releases/download/v3.18.2/cmake-3.18.2-win64-x64.zip -O cmake-3.18.2-win64-x64.zip && \ + "C:\Program Files\7-zip\7z.exe" x -o$TASKCLUSTER_TASK_DIR/ -tzip -aoa cmake-3.18.2-win64-x64.zip; + rm cmake-*.zip + + wget -q https://bintray.com/boostorg/release/download_file?file_path=1.72.0%2Fbinaries%2Fboost_1_72_0-bin-msvc-all-32-64.7z -O boost_1_72_0-bin-msvc-all-32-64.7z && \ + "C:\Program Files\7-zip\7z.exe" x -o$TASKCLUSTER_TASK_DIR/ -t7z -aoa boost_1_72_0-bin-msvc-all-32-64.7z; + rm boost_*.7z + + wget -q https://tukaani.org/xz/xz-5.2.5-windows.7z -O xz-5.2.5-windows.7z && \ + "C:\Program Files\7-zip\7z.exe" x -o$TASKCLUSTER_TASK_DIR/xz-5.2.5/ -t7z -aoa xz-5.2.5-windows.7z; + rm xz-*.7z + + wget -q https://github.com/philr/bzip2-windows/releases/download/v1.0.8.0/bzip2-dev-1.0.8.0-win-x64.zip -O bzip2-dev-1.0.8.0-win-x64.zip && \ + "C:\Program Files\7-zip\7z.exe" x -o$TASKCLUSTER_TASK_DIR/bzip2-dev-1.0.8.0-win-x64/ -tzip -aoa bzip2-dev-1.0.8.0-win-x64.zip; + rm bzip2-dev-*.zip + ;; + esac + + git submodule --quiet sync kenlm/ && git submodule --quiet update --init kenlm/ + + export EIGEN3_ROOT=${DS_ROOT_TASK}/DeepSpeech/ds/eigen-3.3.7 + wget -q -O - https://gitlab.com/libeigen/eigen/-/archive/3.3.7/eigen-3.3.7.tar.bz2 |tar xj +popd diff --git a/taskcluster/kenlm_win-amd64-cpu-opt.yml.DISABLED b/taskcluster/kenlm_win-amd64-cpu-opt.yml.DISABLED new file mode 100644 index 00000000..6c5d5fd4 --- /dev/null +++ b/taskcluster/kenlm_win-amd64-cpu-opt.yml.DISABLED @@ -0,0 +1,12 @@ +build: + template_file: generic_tc_caching-win-opt-base.tyml + cache: + artifact_url: ${system.kenlm.win_amd64_cpu.url} + artifact_namespace: ${system.kenlm.win_amd64_cpu.namespace} + scripts: + setup: "taskcluster/kenlm_tc-setup.sh --windows-amd64" + build: "taskcluster/kenlm_tc-build.sh --windows-amd64" + package: "taskcluster/kenlm_tc-package.sh" + metadata: + name: "KenLM Windows AMD64 CPU" + description: "Building KenLM for Windows/AMD64, CPU only, optimized version" diff --git a/taskcluster/tc-all-utils.sh b/taskcluster/tc-all-utils.sh index defc7350..2e8d0d76 100755 --- a/taskcluster/tc-all-utils.sh +++ b/taskcluster/tc-all-utils.sh @@ -19,9 +19,29 @@ generic_download_tarxz() ${WGET} ${url} -O - | ${UNXZ} | ${TAR} -C ${target_dir} -xf - } +generic_download_targz() +{ + target_dir=$1 + url=$2 + + if [ -z "${target_dir}" -o -z "${url}" ]; then + echo "Empty name for target directory or URL:" + echo " target_dir=${target_dir}" + echo " url=${url}" + exit 1 + fi; + + mkdir -p ${target_dir} || true + + ${WGET} ${url} -O - | ${UNGZ} | ${TAR} -C ${target_dir} -xf - +} + download_native_client_files() { - generic_download_tarxz "$1" "${DEEPSPEECH_ARTIFACTS_ROOT}/native_client.tar.xz" + local _target_dir=$1 + local _nc_url=$(get_dependency_url "native_client.tar.xz") + + generic_download_tarxz "${_target_dir}" "${_nc_url}" } set_ldc_sample_filename() diff --git a/taskcluster/tc-all-vars.sh b/taskcluster/tc-all-vars.sh index 47baf1a1..178d486a 100755 --- a/taskcluster/tc-all-vars.sh +++ b/taskcluster/tc-all-vars.sh @@ -63,6 +63,7 @@ WGET=${WGET:-"wget"} TAR=${TAR:-"tar"} XZ=${XZ:-"pixz -9"} UNXZ=${UNXZ:-"pixz -d"} +UNGZ=${UNGZ:-"gunzip"} if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then WGET=/usr/bin/wget.exe diff --git a/taskcluster/tc-android-ds-tests.sh b/taskcluster/tc-android-ds-tests.sh index c0f2d36e..9a9ba347 100755 --- a/taskcluster/tc-android-ds-tests.sh +++ b/taskcluster/tc-android-ds-tests.sh @@ -14,14 +14,6 @@ model_source=${DEEPSPEECH_TEST_MODEL//.pb/.tflite} model_name=$(basename "${model_source}") export DATA_TMP_DIR=${ANDROID_TMP_DIR}/ds -if [ "${arm_flavor}" = "armeabi-v7a" ]; then - export DEEPSPEECH_ARTIFACTS_ROOT=${DEEPSPEECH_ARTIFACTS_ROOT_ARMV7} -fi - -if [ "${arm_flavor}" = "arm64-v8a" ]; then - export DEEPSPEECH_ARTIFACTS_ROOT=${DEEPSPEECH_ARTIFACTS_ROOT_ARM64} -fi - download_material "${TASKCLUSTER_TMP_DIR}/ds" android_start_emulator "${arm_flavor}" "${api_level}" diff --git a/taskcluster/tc-android-utils.sh b/taskcluster/tc-android-utils.sh index 752d5765..8f4eb69c 100755 --- a/taskcluster/tc-android-utils.sh +++ b/taskcluster/tc-android-utils.sh @@ -119,7 +119,7 @@ android_setup_emulator() fi export PATH=${ANDROID_SDK_HOME}/tools/bin/:${ANDROID_SDK_HOME}/platform-tools/:$PATH - export DS_BINARY_PREFIX="adb shell LD_LIBRARY_PATH=${ANDROID_TMP_DIR}/ds/ ${ANDROID_TMP_DIR}/ds/" + export DS_BINARY_PREFIX="adb shell TMP=${ANDROID_TMP_DIR}/ LD_LIBRARY_PATH=${ANDROID_TMP_DIR}/ds/ ${ANDROID_TMP_DIR}/ds/" # Pipe yes in case of license being shown yes | sdkmanager --update @@ -141,7 +141,7 @@ android_start_emulator() local _api_level=${2:-android-25} export PATH=${ANDROID_SDK_HOME}/tools/bin/:${ANDROID_SDK_HOME}/platform-tools/:$PATH - export DS_BINARY_PREFIX="adb shell LD_LIBRARY_PATH=${ANDROID_TMP_DIR}/ds/ ${ANDROID_TMP_DIR}/ds/" + export DS_BINARY_PREFIX="adb shell TMP=${ANDROID_TMP_DIR}/ LD_LIBRARY_PATH=${ANDROID_TMP_DIR}/ds/ ${ANDROID_TMP_DIR}/ds/" # minutes (2 minutes by default) export ADB_INSTALL_TIMEOUT=8 diff --git a/taskcluster/tc-cpp_tflite-tests-prod.sh b/taskcluster/tc-cpp_tflite-tests-prod.sh index 5acd4016..e1b98a5d 100644 --- a/taskcluster/tc-cpp_tflite-tests-prod.sh +++ b/taskcluster/tc-cpp_tflite-tests-prod.sh @@ -11,7 +11,6 @@ model_source=${DEEPSPEECH_PROD_MODEL//.pb/.tflite} model_name=$(basename "${model_source}") model_name_mmap=$(basename "${model_source}") model_source_mmap=${DEEPSPEECH_PROD_MODEL_MMAP//.pbmm/.tflite} -export DEEPSPEECH_ARTIFACTS_ROOT=${DEEPSPEECH_ARTIFACTS_TFLITE_ROOT} export DATA_TMP_DIR=${TASKCLUSTER_TMP_DIR} download_material "${TASKCLUSTER_TMP_DIR}/ds" diff --git a/taskcluster/tc-cpp_tflite_basic-ds-tests.sh b/taskcluster/tc-cpp_tflite_basic-ds-tests.sh index 7370e8a3..adce0daf 100644 --- a/taskcluster/tc-cpp_tflite_basic-ds-tests.sh +++ b/taskcluster/tc-cpp_tflite_basic-ds-tests.sh @@ -9,7 +9,6 @@ set_ldc_sample_filename "${bitrate}" model_source=${DEEPSPEECH_TEST_MODEL//.pb/.tflite} model_name=$(basename "${model_source}") -export DEEPSPEECH_ARTIFACTS_ROOT=${DEEPSPEECH_ARTIFACTS_TFLITE_ROOT} export DATA_TMP_DIR=${TASKCLUSTER_TMP_DIR} download_material "${TASKCLUSTER_TMP_DIR}/ds" diff --git a/taskcluster/tc-node_tflite-tests-prod.sh b/taskcluster/tc-node_tflite-tests-prod.sh index f1cf01d9..2fd4387c 100644 --- a/taskcluster/tc-node_tflite-tests-prod.sh +++ b/taskcluster/tc-node_tflite-tests-prod.sh @@ -18,7 +18,6 @@ model_source=${DEEPSPEECH_PROD_MODEL//.pb/.tflite} model_name=$(basename "${model_source}") model_name_mmap=$(basename "${model_source}") model_source_mmap=${DEEPSPEECH_PROD_MODEL_MMAP//.pbmm/.tflite} -export DEEPSPEECH_ARTIFACTS_ROOT=${DEEPSPEECH_ARTIFACTS_TFLITE_ROOT} download_data diff --git a/taskcluster/tc-py-utils.sh b/taskcluster/tc-py-utils.sh index 36f80731..6d573c20 100755 --- a/taskcluster/tc-py-utils.sh +++ b/taskcluster/tc-py-utils.sh @@ -271,16 +271,11 @@ get_python_pkg_url() pkgname="deepspeech" fi - local root=$4 - if [ -z "${root}" ]; then - root="${DEEPSPEECH_ARTIFACTS_ROOT}" - fi - local platform=$(python -c 'import sys; import platform; plat = platform.system().lower(); arch = platform.machine().lower(); plat = "manylinux1" if plat == "linux" and arch == "x86_64" else plat; plat = "macosx_10_10" if plat == "darwin" else plat; plat = "win" if plat == "windows" else plat; sys.stdout.write("%s_%s" % (plat, platform.machine().lower()));') local whl_ds_version="$(python -c 'from pkg_resources import parse_version; print(parse_version("'${DS_VERSION}'"))')" local deepspeech_pkg="${pkgname}-${whl_ds_version}-cp${pyver_pkg}-cp${pyver_pkg}${py_unicode_type}-${platform}.whl" - echo "${root}/${deepspeech_pkg}" + echo "$(get_dependency_url ${deepspeech_pkg})" } get_tflite_python_pkg_name() diff --git a/taskcluster/tc-python_tflite-tests-prod.sh b/taskcluster/tc-python_tflite-tests-prod.sh index 5f73484c..3302538b 100644 --- a/taskcluster/tc-python_tflite-tests-prod.sh +++ b/taskcluster/tc-python_tflite-tests-prod.sh @@ -14,10 +14,6 @@ model_name=$(basename "${model_source}") model_name_mmap=$(basename "${model_source}") model_source_mmap=${DEEPSPEECH_PROD_MODEL_MMAP//.pbmm/.tflite} -if [ ! -z "${DEEPSPEECH_ARTIFACTS_TFLITE_ROOT}" ]; then - export DEEPSPEECH_ARTIFACTS_ROOT=${DEEPSPEECH_ARTIFACTS_TFLITE_ROOT} -fi; - download_data maybe_setup_virtualenv_cross_arm "${pyalias}" "deepspeech" diff --git a/taskcluster/tc-python_tflite-tests.sh b/taskcluster/tc-python_tflite-tests.sh index 7f00ea12..fa087bb0 100644 --- a/taskcluster/tc-python_tflite-tests.sh +++ b/taskcluster/tc-python_tflite-tests.sh @@ -14,10 +14,6 @@ model_name=$(basename "${model_source}") model_name_mmap=$(basename "${model_source}") model_source_mmap=${DEEPSPEECH_PROD_MODEL_MMAP//.pbmm/.tflite} -if [ ! -z "${DEEPSPEECH_ARTIFACTS_TFLITE_ROOT}" ]; then - export DEEPSPEECH_ARTIFACTS_ROOT=${DEEPSPEECH_ARTIFACTS_TFLITE_ROOT} -fi; - download_data maybe_setup_virtualenv_cross_arm "${pyalias}" "deepspeech" diff --git a/taskcluster/tc-scorer-tests.sh b/taskcluster/tc-scorer-tests.sh new file mode 100644 index 00000000..9070af39 --- /dev/null +++ b/taskcluster/tc-scorer-tests.sh @@ -0,0 +1,109 @@ +#!/bin/bash + +set -xe + +source $(dirname "$0")/tc-tests-utils.sh + +kenlm_url=$1 + +maybe_py_or_android="$2" +maybe_android="" +maybe_py="" +if [ ! -z "${maybe_py_or_android}" -a "${maybe_py_or_android}" != "android" ]; then + maybe_py=${maybe_py_or_android} + extract_python_versions "${maybe_py}" "pyver" "pyver_pkg" "py_unicode_type" "pyconf" "pyalias" +fi + +if [ "${maybe_py_or_android}" = "android" ]; then + maybe_android="y" + arm_flavor=$3 + api_level=$4 +fi; + +mkdir -p ${TASKCLUSTER_ARTIFACTS} || true +mkdir -p ${TASKCLUSTER_TMP_DIR}/scorer/bins ${TASKCLUSTER_TMP_DIR}/scorer/data || true + +generic_download_targz ${TASKCLUSTER_TMP_DIR}/scorer/bins/ "${kenlm_url}" +download_native_client_files ${TASKCLUSTER_TMP_DIR}/scorer/bins/ + +export PATH=${TASKCLUSTER_TMP_DIR}/scorer/bins/:$PATH + +if [ ! -z "${pyalias}" ]; then + maybe_setup_virtualenv_cross_arm "${pyalias}" "deepspeech" + virtualenv_activate "${pyalias}" "deepspeech" +fi + +if [ "${maybe_android}" = "y" ]; then + android_start_emulator "${arm_flavor}" "${api_level}" +fi + +pushd ${DS_DSDIR} + SCORER_DATA_DIR=${TASKCLUSTER_TMP_DIR}/scorer/data + + cp data/smoke_test/vocab.txt ${SCORER_DATA_DIR} + cp data/smoke_test/vocab.txt ${SCORER_DATA_DIR}/vocab-500.txt + cp data/alphabet.txt ${SCORER_DATA_DIR} + gzip ${SCORER_DATA_DIR}/vocab.txt + + if [ "${maybe_android}" = "y" ]; then + adb shell mkdir ${ANDROID_TMP_DIR}/ds/ ${ANDROID_TMP_DIR}/scorer/ + adb push ${SCORER_DATA_DIR}/ ${ANDROID_TMP_DIR}/scorer/ + adb push ${TASKCLUSTER_TMP_DIR}/scorer/bins/* ${ANDROID_TMP_DIR}/ds/ + + SCORER_DATA_DIR=${ANDROID_TMP_DIR}/scorer/data + fi + + if [ ! -z "${maybe_py}" ]; then + pip install --only-binary :all: progressbar2 + + python data/lm/generate_lm.py \ + --input_txt ${SCORER_DATA_DIR}/vocab.txt.gz \ + --output_dir ${SCORER_DATA_DIR}/ \ + --top_k 500 \ + --kenlm_bins ${TASKCLUSTER_TMP_DIR}/scorer/bins/ \ + --arpa_order 5 \ + --max_arpa_memory "85%" \ + --arpa_prune "0|0|1" \ + --binary_a_bits 255 \ + --binary_q_bits 8 \ + --binary_type trie + + ls -hal ${SCORER_DATA_DIR} + fi + + if [ "${maybe_android}" = "y" ]; then + ${DS_BINARY_PREFIX}lmplz \ + --memory 64M \ + --order 2 \ + --discount_fallback \ + --text ${SCORER_DATA_DIR}/vocab-500.txt \ + --arpa ${SCORER_DATA_DIR}/lm.arpa + + ${DS_BINARY_PREFIX}build_binary \ + -a 255 -q 8 -v trie \ + ${SCORER_DATA_DIR}/lm.arpa \ + ${SCORER_DATA_DIR}/lm.binary + fi + + ${DS_BINARY_PREFIX}generate_scorer_package \ + --alphabet ${SCORER_DATA_DIR}/alphabet.txt \ + --lm ${SCORER_DATA_DIR}/lm.binary \ + --vocab ${SCORER_DATA_DIR}/vocab-500.txt \ + --package ${SCORER_DATA_DIR}/kenlm.scorer \ + --default_alpha 0.5 \ + --default_beta 1.25 + + if [ "${maybe_android}" = "y" ]; then + adb pull ${SCORER_DATA_DIR}/kenlm.scorer ${TASKCLUSTER_TMP_DIR}/scorer/data/ + fi + + ls -hal ${TASKCLUSTER_TMP_DIR}/scorer/data/kenlm.scorer +popd + +if [ ! -z "${pyalias}" ]; then + virtualenv_deactivate "${pyalias}" "deepspeech" +fi + +if [ "${maybe_android}" = "y" ]; then + android_stop_emulator +fi diff --git a/taskcluster/test-android-opt-base.tyml b/taskcluster/test-android-opt-base.tyml index f1cdba74..4389165f 100644 --- a/taskcluster/test-android-opt-base.tyml +++ b/taskcluster/test-android-opt-base.tyml @@ -33,8 +33,6 @@ then: android_arm64_build: { $eval: as_slugid("android-arm64-cpu-opt") } android_armv7_build: { $eval: as_slugid("android-armv7-cpu-opt") } in: - DEEPSPEECH_ARTIFACTS_ROOT_ARM64: https://community-tc.services.mozilla.com/api/queue/v1/task/${android_arm64_build}/artifacts/public - DEEPSPEECH_ARTIFACTS_ROOT_ARMV7: https://community-tc.services.mozilla.com/api/queue/v1/task/${android_armv7_build}/artifacts/public DEEPSPEECH_TEST_MODEL: https://community-tc.services.mozilla.com/api/queue/v1/task/${training}/artifacts/public/output_graph.pb EXPECTED_TENSORFLOW_VERSION: "${build.tensorflow_git_desc}" @@ -45,7 +43,7 @@ then: - $let: extraSystemSetup: { $eval: strip(str(build.system_setup)) } in: > - apt-get -qq update && apt-get -qq -y install git gnupg pixz sox sudo wget unzip software-properties-common xvfb && + apt-get -qq update && apt-get -qq -y install curl git gnupg pixz python-simplejson sox sudo wget unzip software-properties-common xvfb && add-apt-repository --yes ppa:openjdk-r/ppa && apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get -qq -y --force-yes install openjdk-8-jdk && java -version && update-ca-certificates -f && ${extraSystemSetup} && chmod 777 /dev/kvm && adduser --system --home ${system.homedir.linux} ${system.username} && diff --git a/taskcluster/test-armbian-opt-base.tyml b/taskcluster/test-armbian-opt-base.tyml index afe7b64b..f9aec02a 100644 --- a/taskcluster/test-armbian-opt-base.tyml +++ b/taskcluster/test-armbian-opt-base.tyml @@ -26,7 +26,6 @@ then: training: { $eval: as_slugid(build.test_model_task) } linux_arm64_build: { $eval: as_slugid("linux-arm64-cpu-opt") } in: - DEEPSPEECH_ARTIFACTS_ROOT: https://community-tc.services.mozilla.com/api/queue/v1/task/${linux_arm64_build}/artifacts/public DEEPSPEECH_TEST_MODEL: https://community-tc.services.mozilla.com/api/queue/v1/task/${training}/artifacts/public/output_graph.pb DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pb DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pbmm diff --git a/taskcluster/test-darwin-opt-base.tyml b/taskcluster/test-darwin-opt-base.tyml index 1df6b95f..87755b02 100644 --- a/taskcluster/test-darwin-opt-base.tyml +++ b/taskcluster/test-darwin-opt-base.tyml @@ -30,8 +30,6 @@ then: darwin_amd64_build: { $eval: as_slugid("darwin-amd64-cpu-opt") } darwin_amd64_tflite: { $eval: as_slugid("darwin-amd64-tflite-opt") } in: - DEEPSPEECH_ARTIFACTS_ROOT: https://community-tc.services.mozilla.com/api/queue/v1/task/${darwin_amd64_build}/artifacts/public - DEEPSPEECH_ARTIFACTS_TFLITE_ROOT: https://community-tc.services.mozilla.com/api/queue/v1/task/${darwin_amd64_tflite}/artifacts/public DEEPSPEECH_TEST_MODEL: https://community-tc.services.mozilla.com/api/queue/v1/task/${training}/artifacts/public/output_graph.pb DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pb DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pbmm diff --git a/taskcluster/test-generate_scorer-android-24-arm64-opt.yml b/taskcluster/test-generate_scorer-android-24-arm64-opt.yml new file mode 100644 index 00000000..2a85fce1 --- /dev/null +++ b/taskcluster/test-generate_scorer-android-24-arm64-opt.yml @@ -0,0 +1,15 @@ +build: + template_file: test-android-opt-base.tyml + dependencies: + - "android-arm64-cpu-opt" + - "kenlm_android-arm64-cpu-opt" + - "android-cache-arm64-v8a-android-24" + cache: + url: ${system.android_cache.arm64_v8a.android_24.url} + namespace: ${system.android_cache.arm64_v8a.android_24.namespace} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-scorer-tests.sh ${system.kenlm.android_arm64_cpu.url} android arm64-v8a android-24" + workerType: "${docker.dsTests}" + metadata: + name: "Testing DeepSpeech Android 7.0 ARM64 CPU generate scorer" + description: "Generate a DeepSpeech Scorer for Android 7.0/ARM64, CPU only, optimized version" diff --git a/taskcluster/test-generate_scorer-android-24-armv7-opt.yml b/taskcluster/test-generate_scorer-android-24-armv7-opt.yml new file mode 100644 index 00000000..c2c2ac3b --- /dev/null +++ b/taskcluster/test-generate_scorer-android-24-armv7-opt.yml @@ -0,0 +1,15 @@ +build: + template_file: test-android-opt-base.tyml + dependencies: + - "android-armv7-cpu-opt" + - "kenlm_android-armv7-cpu-opt" + - "android-cache-armeabi-v7a-android-24" + cache: + url: ${system.android_cache.armeabi_v7a.android_24.url} + namespace: ${system.android_cache.armeabi_v7a.android_24.namespace} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-scorer-tests.sh ${system.kenlm.android_armv7_cpu.url} android armeabi-v7a android-24" + workerType: "${docker.dsTests}" + metadata: + name: "Testing DeepSpeech Android 7.0 ARMv7 CPU generate scorer" + description: "Generate a DeepSpeech Scorer for Android 7.0/ARMv7, CPU only, optimized version" diff --git a/taskcluster/test-generate_scorer-android-24-x86_64-opt.yml b/taskcluster/test-generate_scorer-android-24-x86_64-opt.yml new file mode 100644 index 00000000..d2d085d2 --- /dev/null +++ b/taskcluster/test-generate_scorer-android-24-x86_64-opt.yml @@ -0,0 +1,15 @@ +build: + template_file: test-android-opt-base.tyml + dependencies: + - "android-x86_64-cpu-opt" + - "kenlm_android-x86_64-cpu-opt" + - "android-cache-x86_64-android-24" + cache: + url: ${system.android_cache.x86_64.android_24.url} + namespace: ${system.android_cache.x86_64.android_24.namespace} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-scorer-tests.sh ${system.kenlm.android_x86_64_cpu.url} android x86_64 android-24" + workerType: "${docker.dsTests}" + metadata: + name: "Testing DeepSpeech Android 7.0 x86-64 CPU generate scorer" + description: "Generate a DeepSpeech Scorer for Android 7.0/x86-64, CPU only, optimized version" diff --git a/taskcluster/test-generate_scorer-android-25-arm64-opt.yml b/taskcluster/test-generate_scorer-android-25-arm64-opt.yml new file mode 100644 index 00000000..18800884 --- /dev/null +++ b/taskcluster/test-generate_scorer-android-25-arm64-opt.yml @@ -0,0 +1,15 @@ +build: + template_file: test-android-opt-base.tyml + dependencies: + - "android-arm64-cpu-opt" + - "kenlm_android-arm64-cpu-opt" + - "android-cache-arm64-v8a-android-25" + cache: + url: ${system.android_cache.arm64_v8a.android_25.url} + namespace: ${system.android_cache.arm64_v8a.android_25.namespace} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-scorer-tests.sh ${system.kenlm.android_arm64_cpu.url} android arm64-v8a android-25" + workerType: "${docker.dsTests}" + metadata: + name: "Testing DeepSpeech Android 7.1 ARM64 CPU generate scorer" + description: "Generate a DeepSpeech Scorer for Android 7.1/ARM64, CPU only, optimized version" diff --git a/taskcluster/test-generate_scorer-android-25-armv7-opt.yml b/taskcluster/test-generate_scorer-android-25-armv7-opt.yml new file mode 100644 index 00000000..cf4ff68b --- /dev/null +++ b/taskcluster/test-generate_scorer-android-25-armv7-opt.yml @@ -0,0 +1,15 @@ +build: + template_file: test-android-opt-base.tyml + dependencies: + - "android-armv7-cpu-opt" + - "kenlm_android-armv7-cpu-opt" + - "android-cache-armeabi-v7a-android-25" + cache: + url: ${system.android_cache.armeabi_v7a.android_25.url} + namespace: ${system.android_cache.armeabi_v7a.android_25.namespace} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-scorer-tests.sh ${system.kenlm.android_armv7_cpu.url} android armeabi-v7a android-25" + workerType: "${docker.dsTests}" + metadata: + name: "Testing DeepSpeech Android 7.1 ARMv7 CPU generate scorer" + description: "Generate a DeepSpeech Scorer for Android 7.1/ARMv7, CPU only, optimized version" diff --git a/taskcluster/test-generate_scorer-android-25-x86_64-opt.yml b/taskcluster/test-generate_scorer-android-25-x86_64-opt.yml new file mode 100644 index 00000000..bcd0cdc9 --- /dev/null +++ b/taskcluster/test-generate_scorer-android-25-x86_64-opt.yml @@ -0,0 +1,15 @@ +build: + template_file: test-android-opt-base.tyml + dependencies: + - "android-x86_64-cpu-opt" + - "kenlm_android-x86_64-cpu-opt" + - "android-cache-x86_64-android-25" + cache: + url: ${system.android_cache.x86_64.android_25.url} + namespace: ${system.android_cache.x86_64.android_25.namespace} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-scorer-tests.sh ${system.kenlm.android_x86_64_cpu.url} android x86_64 android-25" + workerType: "${docker.dsTests}" + metadata: + name: "Testing DeepSpeech Android 7.1 x86-64 CPU generate scorer" + description: "Generate a DeepSpeech Scorer for Android 7.1/x86-64, CPU only, optimized version" diff --git a/taskcluster/test-generate_scorer-android-26-x86_64-opt.yml b/taskcluster/test-generate_scorer-android-26-x86_64-opt.yml new file mode 100644 index 00000000..cb938cf1 --- /dev/null +++ b/taskcluster/test-generate_scorer-android-26-x86_64-opt.yml @@ -0,0 +1,15 @@ +build: + template_file: test-android-opt-base.tyml + dependencies: + - "android-x86_64-cpu-opt" + - "kenlm_android-x86_64-cpu-opt" + - "android-cache-x86_64-android-26" + cache: + url: ${system.android_cache.x86_64.android_26.url} + namespace: ${system.android_cache.x86_64.android_26.namespace} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-scorer-tests.sh ${system.kenlm.android_x86_64_cpu.url} android x86_64 android-26" + workerType: "${docker.dsTests}" + metadata: + name: "Testing DeepSpeech Android 8.0 x86-64 CPU generate scorer" + description: "Generate a DeepSpeech Scorer for Android 8.0/x86-64, CPU only, optimized version" diff --git a/taskcluster/test-generate_scorer-android-28-x86_64-opt.yml b/taskcluster/test-generate_scorer-android-28-x86_64-opt.yml new file mode 100644 index 00000000..c1bbb9bc --- /dev/null +++ b/taskcluster/test-generate_scorer-android-28-x86_64-opt.yml @@ -0,0 +1,15 @@ +build: + template_file: test-android-opt-base.tyml + dependencies: + - "android-x86_64-cpu-opt" + - "kenlm_android-x86_64-cpu-opt" + - "android-cache-x86_64-android-28" + cache: + url: ${system.android_cache.x86_64.android_28.url} + namespace: ${system.android_cache.x86_64.android_28.namespace} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-scorer-tests.sh ${system.kenlm.android_x86_64_cpu.url} android x86_64 android-28" + workerType: "${docker.dsTests}" + metadata: + name: "Testing DeepSpeech Android 9.0 x86-64 CPU generate scorer" + description: "Generate a DeepSpeech Scorer for Android 9.0/x86-64, CPU only, optimized version" diff --git a/taskcluster/test-generate_scorer-android-29-x86_64-opt.yml b/taskcluster/test-generate_scorer-android-29-x86_64-opt.yml new file mode 100644 index 00000000..e1fd0b11 --- /dev/null +++ b/taskcluster/test-generate_scorer-android-29-x86_64-opt.yml @@ -0,0 +1,15 @@ +build: + template_file: test-android-opt-base.tyml + dependencies: + - "android-x86_64-cpu-opt" + - "kenlm_android-x86_64-cpu-opt" + - "android-cache-x86_64-android-29" + cache: + url: ${system.android_cache.x86_64.android_29.url} + namespace: ${system.android_cache.x86_64.android_29.namespace} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-scorer-tests.sh ${system.kenlm.android_x86_64_cpu.url} android x86_64 android-29" + workerType: "${docker.dsTests}" + metadata: + name: "Testing DeepSpeech Android 10.0 x86-64 CPU generate scorer" + description: "Generate a DeepSpeech Scorer for Android 10.0/x86-64, CPU only, optimized version" diff --git a/taskcluster/test-generate_scorer-android-30-x86_64-opt.yml b/taskcluster/test-generate_scorer-android-30-x86_64-opt.yml new file mode 100644 index 00000000..6d72ffda --- /dev/null +++ b/taskcluster/test-generate_scorer-android-30-x86_64-opt.yml @@ -0,0 +1,15 @@ +build: + template_file: test-android-opt-base.tyml + dependencies: + - "android-x86_64-cpu-opt" + - "kenlm_android-x86_64-cpu-opt" + - "android-cache-x86_64-android-30" + cache: + url: ${system.android_cache.x86_64.android_30.url} + namespace: ${system.android_cache.x86_64.android_30.namespace} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-scorer-tests.sh ${system.kenlm.android_x86_64_cpu.url} android x86_64 android-30" + workerType: "${docker.dsTests}" + metadata: + name: "Testing DeepSpeech Android 11.0 x86-64 CPU generate scorer" + description: "Generate a DeepSpeech Scorer for Android 11.0/x86-64, CPU only, optimized version" diff --git a/taskcluster/test-generate_scorer-darwin-amd64-opt.yml b/taskcluster/test-generate_scorer-darwin-amd64-opt.yml new file mode 100644 index 00000000..b1894121 --- /dev/null +++ b/taskcluster/test-generate_scorer-darwin-amd64-opt.yml @@ -0,0 +1,11 @@ +build: + template_file: test-darwin-opt-base.tyml + dependencies: + - "darwin-amd64-cpu-opt" + - "kenlm_darwin-amd64-cpu-opt" + args: + tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-scorer-tests.sh ${system.kenlm.darwin_amd64_cpu.url} 3.6.10:m" + workerType: "${docker.dsTests}" + metadata: + name: "Testing DeepSpeech macOS AMD64 CPU generate scorer" + description: "Generate a DeepSpeech Scorer for macOS/AMD64, CPU only, optimized version" diff --git a/taskcluster/test-generate_scorer-linux-amd64-opt.yml b/taskcluster/test-generate_scorer-linux-amd64-opt.yml new file mode 100644 index 00000000..9088cd52 --- /dev/null +++ b/taskcluster/test-generate_scorer-linux-amd64-opt.yml @@ -0,0 +1,11 @@ +build: + template_file: test-linux-opt-base.tyml + dependencies: + - "linux-amd64-cpu-opt" + - "kenlm_linux-amd64-cpu-opt" + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-scorer-tests.sh ${system.kenlm.linux_amd64_cpu.url} 3.6.10:m" + workerType: "${docker.dsTests}" + metadata: + name: "Testing DeepSpeech Linux AMD64 CPU generate scorer" + description: "Generate a DeepSpeech Scorer for Linux/AMD64, CPU only, optimized version" diff --git a/taskcluster/test-generate_scorer-linux-arm64-opt.yml b/taskcluster/test-generate_scorer-linux-arm64-opt.yml new file mode 100644 index 00000000..ae9c2bdd --- /dev/null +++ b/taskcluster/test-generate_scorer-linux-arm64-opt.yml @@ -0,0 +1,14 @@ +build: + template_file: test-armbian-opt-base.tyml + dependencies: + - "linux-arm64-cpu-opt" + - "kenlm_linux-arm64-cpu-opt" + system_setup: + > + apt-get -qq -y install ${python.packages_buster.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-scorer-tests.sh ${system.kenlm.linux_arm64_cpu.url} 3.7.6:m" + workerType: "${docker.dsTests}" + metadata: + name: "Testing DeepSpeech Linux ARM64 CPU generate scorer" + description: "Generate a DeepSpeech Scorer for Linux/ARM64, CPU only, optimized version" diff --git a/taskcluster/test-generate_scorer-linux-rpi3-opt.yml b/taskcluster/test-generate_scorer-linux-rpi3-opt.yml new file mode 100644 index 00000000..70a61bab --- /dev/null +++ b/taskcluster/test-generate_scorer-linux-rpi3-opt.yml @@ -0,0 +1,14 @@ +build: + template_file: test-raspbian-opt-base.tyml + dependencies: + - "linux-rpi3-cpu-opt" + - "kenlm_linux-rpi3-cpu-opt" + system_setup: + > + apt-get -qq -y install ${python.packages_buster.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-scorer-tests.sh ${system.kenlm.linux_rpi3_cpu.url} 3.7.6:m" + workerType: "${docker.dsTests}" + metadata: + name: "Testing DeepSpeech Linux RPi3 CPU generate scorer" + description: "Generate a DeepSpeech Scorer for Linux/RPi3, CPU only, optimized version" diff --git a/taskcluster/test-linux-opt-base.tyml b/taskcluster/test-linux-opt-base.tyml index 054f1a52..f4c62bd5 100644 --- a/taskcluster/test-linux-opt-base.tyml +++ b/taskcluster/test-linux-opt-base.tyml @@ -28,8 +28,6 @@ then: linux_amd64_tflite: { $eval: as_slugid("linux-amd64-tflite-opt") } linux_amd64_ctc: { $eval: as_slugid("linux-amd64-ctc-opt") } in: - DEEPSPEECH_ARTIFACTS_ROOT: https://community-tc.services.mozilla.com/api/queue/v1/task/${linux_amd64_build}/artifacts/public - DEEPSPEECH_ARTIFACTS_TFLITE_ROOT: https://community-tc.services.mozilla.com/api/queue/v1/task/${linux_amd64_tflite}/artifacts/public DEEPSPEECH_TEST_MODEL: https://community-tc.services.mozilla.com/api/queue/v1/task/${training}/artifacts/public/output_graph.pb DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pb DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pbmm diff --git a/taskcluster/test-linux-opt-tag-base.tyml b/taskcluster/test-linux-opt-tag-base.tyml index 631b97c3..b3f084f8 100644 --- a/taskcluster/test-linux-opt-tag-base.tyml +++ b/taskcluster/test-linux-opt-tag-base.tyml @@ -28,8 +28,6 @@ then: linux_amd64_tflite: { $eval: as_slugid("linux-amd64-tflite-opt") } linux_amd64_ctc: { $eval: as_slugid("linux-amd64-ctc-opt") } in: - DEEPSPEECH_ARTIFACTS_ROOT: https://community-tc.services.mozilla.com/api/queue/v1/task/${linux_amd64_build}/artifacts/public - DEEPSPEECH_ARTIFACTS_TFLITE_ROOT: https://community-tc.services.mozilla.com/api/queue/v1/task/${linux_amd64_tflite}/artifacts/public DEEPSPEECH_TEST_MODEL: https://community-tc.services.mozilla.com/api/queue/v1/task/${training}/artifacts/public/output_graph.pb DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pb DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pbmm diff --git a/taskcluster/test-raspbian-opt-base.tyml b/taskcluster/test-raspbian-opt-base.tyml index 2938b6b6..2a37f9ca 100644 --- a/taskcluster/test-raspbian-opt-base.tyml +++ b/taskcluster/test-raspbian-opt-base.tyml @@ -26,7 +26,6 @@ then: training: { $eval: as_slugid(build.test_model_task) } linux_rpi3_build: { $eval: as_slugid("linux-rpi3-cpu-opt") } in: - DEEPSPEECH_ARTIFACTS_ROOT: https://community-tc.services.mozilla.com/api/queue/v1/task/${linux_rpi3_build}/artifacts/public DEEPSPEECH_TEST_MODEL: https://community-tc.services.mozilla.com/api/queue/v1/task/${training}/artifacts/public/output_graph.pb DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pb DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pbmm diff --git a/taskcluster/test-win-cuda-opt-base.tyml b/taskcluster/test-win-cuda-opt-base.tyml index 2094f439..d8353d66 100644 --- a/taskcluster/test-win-cuda-opt-base.tyml +++ b/taskcluster/test-win-cuda-opt-base.tyml @@ -25,7 +25,6 @@ then: training: { $eval: as_slugid(build.test_model_task) } win_amd64_build: { $eval: as_slugid("win-amd64-gpu-opt") } in: - DEEPSPEECH_ARTIFACTS_ROOT: https://community-tc.services.mozilla.com/api/queue/v1/task/${win_amd64_build}/artifacts/public DEEPSPEECH_TEST_MODEL: https://community-tc.services.mozilla.com/api/queue/v1/task/${training}/artifacts/public/output_graph.pb DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pb DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pbmm diff --git a/taskcluster/test-win-opt-base.tyml b/taskcluster/test-win-opt-base.tyml index eba8e5db..55c1aa2e 100644 --- a/taskcluster/test-win-opt-base.tyml +++ b/taskcluster/test-win-opt-base.tyml @@ -26,7 +26,6 @@ then: win_amd64_build: { $eval: as_slugid("win-amd64-cpu-opt") } win_amd64_tflite: { $eval: as_slugid("win-amd64-tflite-opt") } in: - DEEPSPEECH_ARTIFACTS_ROOT: https://community-tc.services.mozilla.com/api/queue/v1/task/${win_amd64_build}/artifacts/public DEEPSPEECH_ARTIFACTS_TFLITE_ROOT: https://community-tc.services.mozilla.com/api/queue/v1/task/${win_amd64_tflite}/artifacts/public DEEPSPEECH_TEST_MODEL: https://community-tc.services.mozilla.com/api/queue/v1/task/${training}/artifacts/public/output_graph.pb DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pb