commit 4c807d68b1c6c9921edb3142c5595aa3d4f8e77a Author: Malory Rose Date: Thu Aug 26 12:05:22 2021 -0700 Initial Commit diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..6257f2e --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,9 @@ +# Microsoft Open Source Code of Conduct + +This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). + +Resources: + +- [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) +- [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) +- Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..bac5191 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,24 @@ +# Contributing + +Welcome, and thank you for your interest in contributing. There are many ways to contribute: +- [Submit issues](https://github.com/microsoft/solution-accelerator-many-models/issues) to report bugs and make suggestions. +- Review the [source code changes](https://github.com/microsoft/solution-accelerator-many-models/pulls). +- Contribute features and fixes by forking the repository and creating a [pull request](https://github.com/microsoft/solution-accelerator-many-models/compare). + + +## Contributor License Agreement + +This project welcomes contributions and suggestions. Most contributions require you to agree to a +Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us +the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com. + +When you submit a pull request, a CLA bot will automatically determine whether you need to provide +a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions +provided by the bot. You will only need to do this once across all repos using our CLA. + + +## Microsoft Open Source Code of Conduct + +This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). +For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or +contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..9e841e7 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ + MIT License + + Copyright (c) Microsoft Corporation. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE diff --git a/NOTICE.txt b/NOTICE.txt new file mode 100644 index 0000000..67ee971 --- /dev/null +++ b/NOTICE.txt @@ -0,0 +1,996 @@ +NOTICES AND INFORMATION +Do Not Translate or Localize + +This software incorporates material from third parties. +Microsoft makes certain open source code available at https://3rdpartysource.microsoft.com, +or you may send a check or money order for US $5.00, including the product name, +the open source component name, platform, and version number, to: + +Source Code Compliance Team +Microsoft Corporation +One Microsoft Way +Redmond, WA 98052 +USA + +Notwithstanding any other terms, you may reverse engineer this software to the extent +required to debug changes to any libraries licensed under the GNU Lesser General Public License. + +--------------------------------------------------------- + +requests 2.26.0 - Apache-2.0 + + +Copyright 2019 Kenneth Reitz +copyright (c) 2012 by Kenneth Reitz. +copyright (c) 2017 by Kenneth Reitz. + +Apache License + +Version 2.0, January 2004 + +http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + + + "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. + + + + "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. + + + + "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + + + + "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. + + + + "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. + + + + "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. + + + + "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). + + + + "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. + + + + "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." + + + + "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: + + (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. + + You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + +To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); + +you may not use this file except in compliance with the License. + +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software + +distributed under the License is distributed on an "AS IS" BASIS, + +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + +See the License for the specific language governing permissions and + +limitations under the License. + +--------------------------------------------------------- + +--------------------------------------------------------- + +idna 2.10 - BSD-2-Clause AND BSD-3-Clause + + + +BSD-2-Clause AND BSD-3-Clause + +--------------------------------------------------------- + +--------------------------------------------------------- + +chardet 4.0.0 - LGPL-2.1-only + + +(c) Bv +(c) CO +(c) BvA +(c) coB +(c) H U3 +(c) Hp X +(c) I3 Y +(c) M2 Y +(c) O3 Y +(c) EY Ij +(c) OY H2 +(c), 1998 +(c) Ca U3a +(c) CaA CO +O-o (c) I3 +(c) C/ 1996 +(c) C/ 2006 +(c) CD Land +(c) MY LIao +(c) Take IT +(c) (c) AAEE +(c) Eac Aaef +(c) u (c) OY +(c) AAeAE WEd +(c) AU Ass CN +(c) AUueAo AI +(c) OAAnPaE I +(c) OY HAySSU +(c) e" (c) SS +(c) o, (c) MY +(c) sY (c) OY +(c) !eI (c) FE +(c) L FWaIoaXs +(c) aa" (c) PS +(c) uAe (c) PY +(c) uu?i 2003A +Copyright 2005 +(c) B Pool Game +(c) CaAU Audegi +(c) lEEe (c) AE +(c) w'Au AASSQY +(c) (c) O1981|~Y +(c) / 1965 Y AEY +(c) AEAAIAo AaAI +(c) nEJW (c) EHI +(c) pAh (c) ASSU +(c) M1960~1980|~Y +(c) aA"AE (c) 1AE +(c) aAua$? (c) HH +(c) uIEC/ (c) EHI +Ee (c) EAEvAACAEe +(c) ,!C2001|~ao!mx +(c) A1/2cC/ (c) EB +(c) Ao2- AIdegPSAo +(c) EDiETCgDi Hiro +Copyright (c) 2005 +Copyright (c) 2006 +Copyright 2004 Mon +Copyright 2004 Sun +Copyright 2005 Thu +Copyright 2005 Tue +Copyright 2005 Wed +Copyright 2006 Mon +(c) *F!ASSUI (c) IY +(c) Au IUAEnCIeAEai +(c) uu?i 1/41o 2003 +CN1o o,? (c) AOdegi +Copyright 2003-2006 +(c) ESSO!A (c) ESSOY +(c) aEaIAua$? (c) EH +(c) e+-AEaoe (c) AEB +AdegeP dege!o (c) Ao +(c) (r)IE bvAbvAi'u'I +(c) AUueAI PEdegdegAo +(c) AUueAo PEdegdegAo +(c) EAeEEEB / CuhAIcI +(c) http://flickr.com +(c) o1998|~07$?eP!?iY +Hx wCnE$?iaeU* (c) Hx +SSea aaaae" (c) Perl. +(c) I3 3QPAZP-'PSY Xao +(c) aueU1nB 2006-01-02 +(c) o*R+-!!B$?H (c) EY +(c) 2004 - 2005, Efendi +(c) U. John Battelledeg +(c) aueU1nEB 2006-01-03 +(c) degO!A*iuM!O (c) MY +Copyright (c) 2004-2005 +(c) $?IIo E PEa!U1993.2. +Copyright (c) 2006, AND0 +(c) Ao ,oCN'U Ao COdegauE +(c) EREaC/-eAEC/$? (c) RL +(c) uu?i 1/41o 2003A Intel +copyright BBCHungarian.com +(c) ,oua http://sosa0sa.com +(c) BJoel on SoftwareICgEaA +(c) CIAEdeg MSAC Classified +Copyright (c) 2005 Carshops +(c) ,SSCI'A http://kysky.com +(c) a http://konkurs.susu.ru +Copyright (c) 2006, Grebeweb +(c) $?cAAAzEnPaEC/I (c) HAASS +(c) *-deg!Ao ?PoAI degOAOA CO +(c) lPi$?JotAAdege!A1962|~otY +(c) Lionhard Technologies 2003 +(c) g(r)R'Nn?o$?F!I!v!A (c) OY +(c) o a http://www.pots.com.tw +(c) http://blog.mlmaster.com/?p +(c) http://susu.ac.ru/gerb2.gif +Copyright 2001, Nikolay Hristov +Movable Type Copyright (c) 2005 +Movable Type Copyright (c) 2006 +(c) 1/4<>D (c) PO2 +(c) ?idegE http://klutzy.x-y.net +(c) http://ch.kitaguni.tv/u/8280 +Copyright (c) 2005 AmbitUSA Inc. +(c) Copyright 2005, Sharks.co.il. +(c) o1/4!Y http://artifact-jp.com +(c) B OEo1/2, L-IEzIaAUCUe Ch bvEm +(c) e e!nPortnoy's Complaint!U1969 +(c) e(r)*PW1L$?@$?d|WdegN P (c) MY +(c) a"C/Y aa"aY a http://susu.ac.ru +(c) ,oua 1/2o+-o KM http://pm2.ww.to +Copyright 2003-2005 A Muvelodes Haza +(c) ,oua http://jungti1234.netcci.net +(c) AU+-a 1/4?i?! ?APodeg! oA3/4ss CO +(c) AUueAo AoY *I PEdegdegAo ?EA AOAo +(c) d$?SSP!'N13!OAy2yao+-D1/2m (c) MY +(c) cao http://www.daihung.com/blog/?p +(c) ao A$?lI!nMidnight's Children!U1981 +Copyright 2005 url http://wordpress.org +(c) R1B!n!@The Nature and Destiny of Man +Copyright 2005 Dow Jones & Company, Inc. +Copyright (c) 1998 the Initial Developer. +Copyright (c) 2001 the Initial Developer. +Copyright (c) 2005 the Initial Developer. +(c) cuAAA / http://px.a8.net/svt/ejp?a8mat +(c) eEue1/2 http://px.a8.net/svt/ejp?a8mat +(c) ae ol li a http://www.tipilp.susu.ac.ru +Copyright (c) 2006 Herczeg Jozsef Tamas Wed +copyrighted by the Free Software Foundation +(c) 1/21... REBOOTED http://andore.com/money +(c) C*P"Y !C http://willythecop.blogspot.com +(c) http://kapranoff.ru/archives/003650.html +(c) oW3Ooo' W3Ooo http://www.coolloud.org.tw +Copyright 2003, Greenline, Kazan, Russia Wed +(c) !E http://blog.livedoor.jp/facilitators/' +(c) "aae http://money.rin.ru/content/news/?id +(c) B http://sasuga.biz/pages/index.php?refid +(c) http://azoz.org/archives/200512031633.php +(c) (c) AA+-C/IAtBGCg http://www.1affliate.com +(c) +-,degO AIu?Ca http://deholexp.mizc.com/wp +(c) EEEEBU1/2V1/2ENEu-1/2u"~XgvAaie$? (c) EAEB +(c) !PS ODE$?uAECPS!Google 1OuODgooglesucks.com +(c) 3/4ss CN'U... http://kina.egloos.com/655614 +(c) C/AE'PeBOExI http://px.a8.net/svt/ejp?a8mat +(c) CaA AE'A, *13/4ioIA degCA(r)?A degI ?U?! CO +(c) cIpibx http://www.mag2.com/m/0000103697.htm +(c) http://www.acnnewswire.net/Article.Asp?lang +(c) ua CCAU cA* http://xenix.egloos.com/1213811 +(c) *C/AEd http://cosoft.org.cn/projects/webpm/' +(c) 1/2A'U http://chisato.info/blog/index.php?pl +(c) AO,c u?E3 ?eA ue*AAO degIAI'U. +-x*A3/4ss CO +(c) cC$?1/4 SRC http://inkase.net/if/docomo.html +(c) cC$?1/4o http://www.motionlink.jp/clk.php?pt +(c) http://andore.com/money/archives/003398.html +(c) jao C!K!nThe Prime of Miss Jean Brodie!U1961 +(c) o-d$?F$?@|~|haoao3/4v*P"i D+-'11/2*D!A (c) OY +Copyright (c) 2005, CigarMinds Kft. 2006. 01. 04. +Copyright 2005 rdf:resource http://blog.empas.com +(c) A$?AE+-eEnA*aAu http://px.a8.net/svt/ejp?a8mat +(c) O| http://www.acnnewswire.net/Article.Asp?lang +(c) *-deg!Ao http://jowchung.oolim.net/index.php?pl +(c) Oa,o*thInuAIoO3/4EC http://www.messagecast.net' +(c) a(r)$?Y aa http://money.rin.ru/content/news/?id +(c) http://nanana.moo.jp/archives/000922.html Y ThY +(c) u1AoCI degIAI AIdegPSAC ,A*AI+-auu CO degIAI'U. +(c) a http://www.greek.ru/ru/news/news_detail.php?ID +(c) cC$?1/4I http://sasuga.biz/pages/index.php?refid +(c) http://www.newsru.com/crime/03jan2006/minsk.html +(c) a? http://andore.com/inami/mtarchives/003381.html +(c) eaOOc, a"o"O http://money.rin.ru/content/news/?id +(c) u2oAuIa http://club.h14m.org/kenji/diary/?200512b +(c) ITOIA*B / aAIeeEI http://www.wimp.com/rubberjohnny +(c) http://www.newsru.com/crime/03jan2006/germanz.html +(c) aIuOOINat< http://webryblog.biglobe.ne.jp/'> +Copyright (c) 1991, 1999 Free Software Foundation, Inc. +(c) $?AE http://mimizun.com:81/blog/archives/000314.html +(c) I"OAFaith3n / http://www.finechixxx.com/galleries/61 +(c) c http://www.topre.co.jp/products/comp/key_list.html +(c) deg! i3/4u3/4i?? http://zangsalang.egloos.com/594634 +(c) e"|O"eOO" OconAE DiOO" *o| O*" http://music.peeps.ru +(c) |uA1/2IAd*uaC/eBgp^CI http://px.a8.net/svt/ejp?a8mat +(c) A |ae http://www.ebao.us/portal/showcontent.asp?INDEX +(c) EIC/~UcEC/B http://andore.com/mt/archives/003192.html +(c) cC$?1/4@ < /> http://www.accesstrade.net/at/c.html?rk +(c) e http://webryblog.biglobe.ne.jp/2/d/2d5396c6b9.html' +(c) e http://webryblog.biglobe.ne.jp/7/5/75575704c2.html' +(c) oSS http://music.peeps.ru/news/s/2003/03/12/1366.html +copyright 2015, Mark Pilgrim, Dan Blanchard, Ian Cordasco +(c) +-a CNAth ASSAO'I'U. http://epitaph.egloos.com/1106816 +(c) $?A3/4a$?2$?e http://club.h14m.org/kenji/diary/?200512b +(c) A1/2 $?A*B / http://www.monsterhunter.us/beastof7chutes +(c) c http://itpro.nikkeibp.co.jp/free/NT/NEWS/20050225/1/' +(c) !x!O http://www.aozora.gr.jp/cards/000050/card1174.html' +(c) AEnT~I..... http://andore.com/money/archives/003302.html +(c) a !- "" http://www.newsru.com/russia/03jan2006/gruz.html +(c) ae O!o!SS http://www.greek.ru/ru/news/news_detail.php?ID +(c) http://www.bphrs.net/mesi/archives/2005/04/post_110.html +(c) http://www.bphrs.net/mesi/archives/2005/07/post_116.html +(c) uu?i XP*I AE/,aCO http://jowchung.oolim.net/index.php?pl +(c) |AAGC/aeFaith3n / http://www.finechixxx.com/galleries/65 +(c) N3/4-CAZNV GvdaIuAeAC/e http://ore-1gpy.cocolog-nifty.com +(c) a http://www.amefoot.net/archives/2005/05/13_000876.html' +(c) eAEAu http://fudemame-guide.com/fude15/faq/fmF15106.html' +(c) http://www.pinkupa.com/cgi-bin/weblog/archives/000611.php +(c) http://www.pinkupa.com/cgi-bin/weblog/archives/000616.php +(c) +-,AC 3/4o+-1/4AIdeg!? http://zangsalang.egloos.com/592609 +(c) ,(r)1/2oA A-'O http://www.acnnewswire.net/Article.Asp?lang +(c) cC$?1/4@ < /> http://click.linksynergy.com/fs-bin/click?id +(c) eAC/U*B / http://www.businessnetwork.co.jp/HTM/choki.html' +(c) http://contents-factory.com/blog/archives/200511301334.php +(c) http://furusatonoeki.cutegirl.jp/main/archives/001172.html +(c) l+-u"u3o|W http://www.ebao.us/portal/showcontent.asp?INDEX +(c) uAc 05/12/26@ http://blog.inkase.net/2005/12/html_404.html +(c) "aae " (r)aY aa(r)C/ea http://money.rin.ru/content/news/?id +(c) 13/4!O http://www.aozora.gr.jp/cards/000275/card45513.html' +(c) 13/4!O http://www.aozora.gr.jp/cards/000275/card45514.html' +(c) AAw http://affiliate-school.com/products/letter.html target +(c) albgAaeEEA1/2eauAecuC/Bu+-$?IEl3na!RAaiAAenPaEC/I (c) EEEEB +(c) !E http://www.pinkupa.com/cgi-bin/weblog/archives/000612.php +(c) 1OOUWikiuAIoA?PS!xOE http://ccca.nctu.edu.tw/~hlb/tavi/WiKi' +(c) EH http://tamuyou.haun.org/mt/archives/2006/01/post_383.html +(c) PSC/ a$?"" http://www.aviaport.ru/news/2006/01/02/99276.html +(c) !a http://furusatonoeki.cutegirl.jp/main/archives/001177.html +(c) E<<*1/2I>>+-",ae http://www.coverer.com/archives/000830.shtml +(c) IA3!C*U!O|31/2i!C http://www.upsaid.com/isis/index.php?action +(c) Lionhardt Technologies 2003, www.lionhardt.com/bb Wed, 04 Jan +(c) O'1N!3/4N21O I?uiaCAia* http://www.opentle.org/modules.php?op +(c) e 'a!AdegN P 'aY AP!oE1iY PTAp*u! Hong Kong People's Alliance +(c) 1/4ONuE1/2 http://www.w3cn.org/article/translate/2005/115.html +(c) Ac?I $?E$?I!C/ http://iriz.hanazono.ac.jp/frame/k_room_f1.html +(c) cCAP*eAE?a3?EEeU*aeB / http://www.accesstrade.net/at/c.html?rk +(c) (c) AA+-C/IAtBGCgxI j AaaAAEIieUu1/2B http://www.1affilaite.com +(c) 3U!A http://furusatonoeki.cutegirl.jp/main/archives/001182.html +(c) eU*B src http://clickablewords.com/archives/img/yomo_051020.jpg +(c) cEEEEE http://tamuyou.haun.org/mt/archives/2005/12/post_379.html +(c) uAA!EINuE1/2AD+-i http://www.w3cn.org/article/tips/2005/116.html +(c) IoA3/4!E !C/ http://www.aozora.gr.jp/cards/001154/card44333.html' +(c) M http://www.literature.org/authors/carroll-lewis/' Lewis Carroll +(c) aueEC/a http://tamuyou.haun.org/mt/archives/2005/12/post_377.html +(c) c2EEeU*B / https://ssl.hosting-link.ne.jp/adbyclick.asp?adurlname +(c) http://forum.template-toolkit.ru/view_topic/topic_id-115.html?rss +(c) deg!uE+-i? http://jely.pe.kr/archives/2004/10/20041021_000236.html +(c) o3oY /ICASHao$?oSSta Y iY H"PS a http://www.wretch.cc/blog/iamryan +(c) *-deg!Ao 1/4on1/2o,| AC/COo,'I title http://xenix.egloos.com/189169' +(c) ,| c?eCO1/4o AOdegOuE'U. http://www.acnnewswire.net/Article.Asp?lang +(c) 1Uua$?B src http://www.beginnersrack.com/mt/images/20050429_soba.jpg +(c) C/eC/eEiTha eY1/2C/A*aAGGIEI http://nefdesfous.free.fr/sculpture.htm +(c) cA Cga*aoeTCgEPUeIiA1/2u http://tanoshi.chance.com/reg_tanoshi.php?I +(c) cC$?1/4@ < /> http://ck.jp.ap.valuecommerce.com/servlet/referral?sid +(c) uA1OOUGoogleOA -uADAIC/ http://www.coverer.com/archives/000832.shtml +(c) Ca http://blog.bd-lab.com/blog/archives/000130.html Ao3 +-Y?aAI Au3a. +(c) Ca!ae,N Yxaae!iC, !o! O! eCa!,Oe http://money.rin.ru/content/news/?id +(c) A1/2 http://www.cycle-yoshida.com/trek/nike/shoes/mtb/5kato3_page.htm' +(c) PS+-PSuoD3/4-1/2/E'$?-$?E$?e http://azoz.org/archives/200511260855.php +(c) Ai1c,|uae1/2e$?IBLOG2oAa http://artifact-jp.com/mt/archives/000472.html +(c) O$?E2n$?A$?AE$?$?Th$?*$?? http://akaname.main.jp/mt/archives/000127.html +Copyright 1999-2004, Slavei Karadjov slaff@linux-bg.org support@linux-bg.org +(c) - (r)e $?" aPS$?"$?" () http://www.newsru.com/world/03jan2006/knifer.html +(c) H@ a?enALb'Aag|U*B uri http://www.sixapart.com/movabletype/' Movable Type +(c) u http://www.coolloud.org.tw/news/database/interface/detailstander.asp?ID +(c) 3OCi IEo1P3AEo*- !x!O http://www.aozora.gr.jp/cards/000042/card42258.html' +Copyright 2005, ACN Newswire corp@asiacorpnet.com tim.mckinnon@asiacorpnet.com +(c) ,oua'O http://archmond.mizc.com/tt target blank http://archmond.mizc.com/tt +(c) 1o,deg AI 1/2AdegPSAo A degE,(r)degUAo,, 1/2AAU,, CN'U,e 1 Ao AndegIdegO CO +(c) Ae,A !x!C/Y EY $?Y e!O http://www.aozora.gr.jp/cards/000009/card45340.html' +(c) C/3/4aAOoI http://www.y-moto.com/bd-1/archives/2005/12/post_267.html'target +(c) I"IauAWi http://www.cnblog.org/blog/archives/2004_08_22_cnblog_archive.html +(c) ID1UA1/2 http://www.cnblog.org/blog/archives/2004_08_22_cnblog_archive.html +(c) O|p|o O3y!H /p p a http://photos1.blogger.com/blogger/2953/388/1600/all.jpg +AAAaeIuIqlAA emIEC/A*lB IUnU1/4EaeIma1/2C/ae$?E'PU*BE'ITCgATlCEEAAeIEnA (c) EeUY +(c) o3C/ OaoY I$?eAO1IY i"PS a http://fudesign.blogspot.com/2005/08/august-8.html +(c) !(r)E3o$?a3/4C1/4u3o3/4E$?a3/4C1/4u!A a http://ilyagram.org/archives/1683.html +(c) ?IPSoODEIOu1/4UOuIThO-AuAPSE1/2 http://blog.westca.com/blog_a/p_full/44725.html +(c) o aSS1I http://www.coolloud.org.tw/news/database/interface/detailstander.asp?ID +(c) o$?@$?e$?@$?e$?W$?E$?Q(r)EA |ae http://www.ebao.us/portal/showcontent.asp?INDEX +(c) oC/wC/w http://www.coolloud.org.tw/news/database/interface/detailstander.asp?ID +(c) 3 3o3/4O'U. 3/4AEAI'o1/2o?!1/4 http://www.inews24.com/php/news_view.php?g_serial +(c) HTML-$?(r)aa!Y a http://forum.template-toolkit.ru/view_topic/topic_id-88.html?rss +(c) c@ http://www.teizouteiki.jp/contact.html' http://www.teizouteiki.jp/contact.html +(c) idegPS W3O1/2x3/4 2005-12-30T18:40:48+08:00 domain http://www.technorati.com/tag' +(c) $?E x$?|$?-$?e$?$?!C/1P1/4e$?E$?aC/II$?C$?*$??!PS src http://ch.kitaguni.tv/u/8280 +(c) ,(r)1/2o,P1/2o 1/4O*I yA,EAE*A degadegu1ssCY http://critique.or.kr/tt/index.php?pl +(c) AU+-a ?i?iCss3/4udego..'U1/2A AC o A1/4AIAa.. .... http://oroll.egloos.com/986804' +(c) C/Y aa(r) Y a $?C/(r)Y -(r)PS"!e"a http://www.newsru.com/world/03jan2006/heli.html +(c) degi3/4uI*PS!u<>NUe$?!PS!P http://www.blogchina.com/new/display/57469.html' +(c) *I1/4OCAAE(r)deg! http://research.microsoft.com/barc/mediapresence/MyLifeBits.aspx' +(c) e+-AEaA U*B" http://www.monsterhunter.us/beastof7chutes/creatureOriginal-message.jpg +(c) nPAB 3/4u(11/24)aCxgaAAeIAiC/1/2C/uIC$?1/4BBB http://orf.sfc.keio.ac.jp/index.html... +(c) o2O'aoaA* |"Y http://www.coolloud.org.tw/news/database/interface/detailstander.asp?ID +(c) u 1/2NY Y oY ?!1/4 http://www.ycf.nanet.co.jp/~dre/cgi-bin/diary/archives/000422.html +(c) mp3A'IyIyPS!*C/IO!PEc1uODO Ii!* 12 'i http://cappuccinos.3322.org/wp-commentsrss2.php?p +(c) iO http://10e.org/mt/archives/200512/280341.php http://anime.livedoor.com/theater/2.html +(c) w+-uoTh1/4OY I!C http://www.coolloud.org.tw/news/database/interface/detailstander.asp?ID +(c) 2uA http://10e.org/mt/archives/200512/280410.php http://10e.org/samcimg3/nekonekoneko.jpg +(c) OeI'uO o1/2th1yEEIauAPS!Oa2AECIOEuO,IyO,IAuA!PS zh rdf:resource http://b2evolution.net/?v +(c) iC/C/ISSvAEvAAYAC/1/2n3/4aA http://www.nisshin.com/life/cm/tvcm_mama0915.html' u't YIAfeB +(c) ieE ssStAEC/$?+-AEAAmeC/AEctOStEsA1/2BsA1/2II http://wa2.e-golf.co.jp/gp/nh/cb/cb133.htm' +Copyright 2006, C IAOOAIDINEEC - D. AEAIAOEAACO OEA A.A. webmaster@naftemporiki.gr Wed, 4 Jan +(c) A IUnUi'Eu AIEuEIAuAEC/A*aeEB src http://www.beginnersrack.com/mt/images/20050811_sumi1.jpg +(c) IAE~(r)uY ~?aAD1qPx http://www.coolloud.org.tw/news/database/interface/detailstander.asp?ID +(c) cC$?1/4o < http://www.cycle-surf.com/?ref 11'> SRC http://inkase.net/images/traffic/cycle.gif +(c) l2x3PS!Y$?PS"i$?deg oaouoaThu2aG!C / http://photos1.blogger.com/blogger/6894/256/1600/??????' +(c) $?oE1/21/4"$?*$??$?$?3/4i1c$?I http://msdn.microsoft.com/workshop/author/VML/ref/appendix.asp' +(c) $?A$?a$?A$?$?$?O OP!$?e$?*$?-$?E$?A$?AE$?$??$?E x$?$?$?Th$?1$?!$?$?$? $?!$?C$?*$?c$?|! (c) PS2Y +(c) c@ http://www.businessnetwork.co.jp/HTM/i-4.html' http://www.businessnetwork.co.jp/HTM/i-4.html +(c) (c) euApPbgaaCEEeU*aeEB"AEIp RaiU+-Ec vEAaA1/2C/E'C/CauAuUC/U*B / http://px.a8.net/svt/ejp?a8mat +(c) PS CUP3O?"OssuAOOuUO 'IOUEO+-3/4IiEUE1OAATMIocuAE"Au http://www.acnnewswire.net/Article.Asp?lang +(c) ?A$?I1/4Pedegi$?I EAI,oEaY iY $?Y o http://www.aivy.co.jp/BLOG_TEST/kobakoba/archives/003047.html +(c) cAFeedBurnerdgA1/2RSSzMEI|AYUu1/2BSubscribe3eAeuI http://feeds.feedburner.com/BragZakatoHeadline' +(c) uAAa*NAEoOuOE1/4th*thInAEPS!?EOOE1OAxO1/4oOoAuIaooxo http://www.coverer.com/archives/000835.shtml +(c) ?IuAAE1/2aPS!Eu3/4I2 auA3oOaNuuA1/2aUAE!PS ?? http://www.blogbus.com/blogbus/blog/index.php?blogid +(c) $?C/"|(r)a a(r)aa! "a-(r) iSSaY aY ?? http://forum.template-toolkit.ru/view_topic/topic_id-114.html?rss +(c) authors@template-toolkit.ru http://forum.template-toolkit.ru/view_topic/topic_id-93.html Template Toolkit +(c) authors@template-toolkit.ru http://forum.template-toolkit.ru/view_topic/topic_id-99.html Template Toolkit +(c) OA1au$?yaoSSU u!nEr redete mit dem Vieh, den Vogeln und den Fischen King Solomon's Ring ^AP King Solomon's +(c) MSSUao$?ssAEFSSe(r)g!CSanwenji !O'2$?aPdegao'P3q,U /u1/4gak(3o,I!U$?FAn1/2O)!C http://sanwenji.blogspot.com +(c) aB!xIa$??aAAElA1/2C/E YNid*eAEC/$?IA$??I1/4eAA1/4eIFlaWUeAEC/$?!-IXOBsA1/2II http://r.gnavi.co.jp/g002234/' +(c) cT*eAEC/$?aIAAutECIoiA3ThiAO YIgp?AuAECIouiypAO3c@OOI@$?CIoia IIUAEEeBv / i http://www.nikkei.co.jp/'target +(c) A $?IPS3PS2oiEE$?!,o3 $?u$?i$??$?3$?E$?E$?E$?e!EdegiI/$?I http://www.aozora.gr.jp/index_pages/person157.html +(c) e1/2J haAnAC/AA20-30amutEjIlDAEeBAEB "qIuEnAHvE|AeecuC/B|ceEC/AEhecuC/B !AXB http://www.rui.jp/ruinet.php3... +(c) !x$?E$?$?$?A$??$?I$? <$?a$?*$?i$?E$?$?!PS /> http://www.city.kyoto.jp/somu/rekishi/fm/ishibumi/html/sa043.html +(c) ,, 1/43/4AI(Berkman Center for Internet & Society)?!1/4 AO+-U title http://cyber.law.harvard.edu/home/home?func +copyright rules http://www.bbc.co.uk/go/wsy/pub/rss/1.0/-/hungarian/learningenglish/witn/2005/05/050527_google.shtml +(c) OA!PEyI IA -O3/4!*dPS?AEuuAIoO*IaPSo http://mag.udn.com/mag/dc/make_rss.jsp' http://mag.udn.com/mag/dc/make_rss.jsp +(c) I2 1,o3oAEO ,oODOaE1/4uAA'1/2OPS!3/4ssIaOUIO PIODEu!PS / / N|A1/4+-oxO1/4oODO ,o http://ccca.nctu.edu.tw/~hlb/tavi/' +(c) OAAauAblogPS!u<>ECEuODa-AAOssP1/4P(r)uAE1 http://www.cnblog.org/blog/archives/2004_08_22_cnblog_archive.html +(c) *I1/4OCAAE(r) cdeg! AIAoCI ?u+-1?!1/4 1/41/4 1odeg*I A 1/4O*c1/4C A|degoAUAI Bytes Technology GroupdeguAC degA*!,| AeCO +(c) PSoOD http://wiki.planetoid.info/index.php/WikiWiki' WikiWiki IoA?oI http://wiki.planetoid.info/index.php/WikiWatchList' +(c) R$?'Y 1/4|"Y ao1/2u$?W1/4s1/41/2!BwebTV1A,O!ANaomi Klein "a1e1DY X"a$?@<>o!A$?u|~*sAIPO a http://video.google.com +(c) cou1/2t erIYeJnBaeI Y^ae@ThIeOfW^ uAEC/AEC/$?+-AEAEA http://www.amazon.co.jp/exec/obidos/ASIN/B000B4NMUA/tamuyoudiary-22/ref +(c) EAE$?a$?AE,'$?1/2P H http://www.coolloud.org.tw/news/database/interface/detailstander.asp?ID 108673 2005|~12$?e14$?e!ASSUI+-u"iY +(c) l$?F2A$?GO!G"AE!M|W|rY s!m http://blog.yam.com/youandme/' "S|3|W|rao"k$?kY D"$? !n |y!O2A$?@O!G"AE!m http://pingfandegushi.blogspot.com/' +(c) 1/2cEC/uA http://ann.269ch.jp/archives/2005/12/google_20.html' n$?cLAiDuO B http://ken2-jp.cocolog-nifty.com/kenken/2005/12/google_cb13.html' +(c) E"IthuADAIC/PS!ECITh*"OuuADAAuuADAIC/A'O'!PSOE1/4thAD+-iuAIOU?EOO2I? http://mail.wikipedia.org/pipermail/wikien-l/2004-August/date.html' OaAi +(c) AE!PS br a http://www.softsea.net/cat/3.htm IuI31$?3/4ss /a a http://www.softsea.net/cat/306.htm 1/2cAae1$?3/4ss /a a http://www.softsea.net/cat/30602.htm +(c) 1/2oDDOth2OIO br a http://www.softsea.net/cat/3.htm IuI31$?3/4ss /a a http://www.softsea.net/cat/305.htm IuI31UAi /a a http://www.softsea.net/cat/30514.htm +(c) assAAaaE AoI~AEEeUu1/2B http://www.moon-light.ne.jp/weblog/archives/2005/12/post_133.html http://www.moon-light.ne.jp/weblog/archives/2005/12/post_133.html +(c) c AgI(r)|I135.4A242.9Au1/2B http://www.moon-light.ne.jp/weblog/archives/2006/01/2005_2.html http://www.moon-light.ne.jp/weblog/archives/2006/01/2005_2.html +(c) PS!+-iIO3oE !PS br a http://www.softsea.net/cat/3.htm IuI31$?3/4ss /a a http://www.softsea.net/cat/303.htm IuI3OoC? /a a http://www.softsea.net/cat/30390.htm +(c) 1n2n?Th1/2n'U$?E$?A$?$?$?AE !C/ http://www.aozora.gr.jp/cards/001166/card43728.html' ?Th1/2n'UE!3U2degIA !C/ http://www.aozora.gr.jp/cards/001166/card43729.html' +(c) PaOOOON-APSE1/2!PS br a http://www.softsea.net/cat/3.htm IuI31$?3/4ss /a a http://www.softsea.net/cat/304.htm deg2E<<*A>>$? /a a http://www.softsea.net/cat/30407.htm +(c) xi'ouA+-PS $?,uEuEuEuE... br a http://www.softsea.net/cat/3.htm IuI31$?3/4ss /a a http://www.softsea.net/cat/303.htm IuI3OoC? /a a http://www.softsea.net/cat/30305.htm +(c) PaOOxa2EEi1/4thIOOEOOA!PS br a http://www.softsea.net/cat/3.htm IuI31$?3/4ss /a a http://www.softsea.net/cat/309.htm 1/4OIY DYID /a a http://www.softsea.net/cat/30912.htm +(c) 3PS1ae1/4AEEa u2Ux/xOP- -1/2oDDIaA?uAuA3IDo br a http://www.softsea.net/cat/3.htm IuI31$?3/4ss /a a http://www.softsea.net/cat/305.htm IuI31UAi /a a http://www.softsea.net/cat/30506.htm +(c) OAAauAblogIaDN*thIn!PSLiveMessageuAooI"EeOAO2*C3PS*a, PS!?EOOIi1/4OPa,oblogPS!O2?EOO,oDO -EeOAIaDNuANuE1/2oI,uDAEuE!PS / / oI http://del.icio.us' Del.icio.us OO1/4deg http://www.furl.net' +(c) PS!OaeEae*th!PSxoIi?aE1/43oAA1/4uAoONPS! 1/2nIiEY +-+-'o!PS http://www.blogcn.com/User3/luciferwang/blog/27037020.html target blank a-AAE IA http://www.blogcn.com/blog/trackback.asp?mydiary +(c) AaAaEC/EC/uAPIIAPS!OaAiAaeN|A1/4+-ouA!P http://ccca.nctu.edu.tw/~hlb/articles/archives/000662.php' Wiki 1/4o1/2e !*?EOO2I?1/4!PSOU!P http://ccca.nctu.edu.tw/~hlb/articles/archives/000662.php' +(c) AyxOPS!AEaODdeguA"PthEO3u!C/3/4U a1ae (r)*thInAeEoOO1/4degO ,oudegdegxOE1/2a11Ey3/4Y?a!PS IO*C/21/4uADAIC/PSo http://base.google.com/base/search?q neckties http://base.google.com/base/search?q +(c) DA!a!aE<>1oUOEAU http://www.blogcn.com/User14/xyl5400/blog/26093815.html target blank a-AAE IA http://www.blogcn.com/blog/trackback.asp?mydiary +(c) 1OIPuAPS!EiOUEoIaODuAEEdegNOaOOIPuA2 PIuA <2Y PS!+-!*C/!PS ?EECPS!3/4?3/41EECE$?Oss!PS http://www.blogcn.com/user3/chen56/blog/431803.html target blank> a-AAE IA http://www.blogcn.com/blog/trackback.asp?mydiary +(c) AO1/2i!PS oOssAuOox D'A1/2*YOOEIuAIAxOIA1/4thPS!AUEYEOOD2iOiPS!u br a http://www.softsea.net/cat/3.htm IuI31$?3/4ss /a a http://www.softsea.net/cat/302.htm IA1/4th1$?3/4ss /a a http://www.softsea.net/cat/30208.htm +(c) IEDDOss1/2 A/O IPS!IyIyEuACECEcoIEIEPWikiuA!PS / / IOAE1/2au1/2uAIO ,oWikiEC http://www.chinesepython.org/cgi_bin/moingb.cgi' ODoth'oOOOo PS!Eu?EAUECOU2002Ae5O*Y?aI"uA!PS http://www.chinesepython.org/cgi_bin/moingb.cgi' +(c) Ae,A !x$?I!C/E!oI!uE$?E$?e$?eEYIo$?C$?C/$?e!PSY EY $?Y e!O http://www.aozora.gr.jp/cards/000009/card45340.html' degA1aeEnAESS?I$?IAEae !x$?IEYIo1/4O$?I!C/ deg3/4a+-/ANuE!E http://www.aozora.gr.jp/index_pages/person726.html +(c) xO!C/OeIOIoPS!1/2a3/4oAEOAGBKeEaEeuAOUBIG5e*PISSAUuAxO*uIoBIG5Oy3PSxa uAIEIa!PSEu br a http://www.softsea.net/cat/3.htm IuI31$?3/4ss /a a http://www.softsea.net/cat/310.htm ODIAIuI3 /a a http://www.softsea.net/cat/31003.htm +(c) !E$?a$?A$?i$?odegaAA OAE$?ss'1$?"$?aIae$?A$?+-$?E!E !!$?E$?a!C/$?a$?A$?i$?o*CoU$?u$?i$?AE$?$?$?e1/4I??$?IAae$?C$?a2 o1/2$?E'O$?1$?e$?a$?I$?I$?a$?A$?i$?o+-oA/$?C$?I$?E$?$?!PSCdeg$?I$??$?a!PS... http://blog.livedoor.jp/safe_food_of_asia +(c) 1/2"OixOP-OEDDEi1/4thuAO OO*1/2E1/2PS!AaOU1/4,,o*OOOOOAUAU1 1/2"OiAauAxO1/4ouA CD/DVD xOP-OEDDEi1/4th!PS br a http://www.softsea.net/cat/3.htm IuI31$?3/4ss /a a http://www.softsea.net/cat/308.htm ?a*C/OAEx/ /a a http://www.softsea.net/cat/30801.htm +(c) D!'iIoPS!degU*OdegU1/4aeEYEI'udegae+-3/4!PS2EOA,uIE1/2ouA?iEUEa*"PS!Ei1/4th1/4aEOdeg2xdeg,u1/4OE Aaex1/4+-,!PS br a http://www.softsea.net/cat/3.htm IuI31$?3/4ss /a a http://www.softsea.net/cat/303.htm IuI3OoC? /a a http://www.softsea.net/cat/30304.htm +(c) OAEuuA1|AUA'uxaAauAIOEaOaCo!PSI!E+-PS!EmEditorECO ,oD!DICa+-a?iEUuAIAEe+-a1/43IE1/2.EuODDiPaC?'ouA1|AUAyEcOA br a http://www.softsea.net/cat/3.htm IuI31$?3/4ss /a a http://www.softsea.net/cat/302.htm IA1/4th1$?3/4ss /a a http://www.softsea.net/cat/30208.htm +(c) 2E$?I OP! !E!C/AIEO21$?!PS+-oiEE!E http://www.aozora.gr.jp/cards/000020/card44722.html' E1/4 I$?E1/2/IY !E!C/E!AA+-N O$?!PS+-oiEE!E http://www.aozora.gr.jp/cards/000057/card43276.html' 3/4a$?IE3/4A,3P !E!C/,PI+-'i$?!PS+-oiEE!E http://www.aozora.gr.jp/cards/000293/card4680.html' +(c) Ae "u !C/ http://www.aozora.gr.jp/cards/000157/card45240.html' 1/4c$?I$?A$?E$?a !C/ http://www.aozora.gr.jp/cards/000157/card45242.html' ?'u!I-EN$?oIA$?o !C/ http://www.aozora.gr.jp/cards/000157/card45243.html' 3/43|$?ED$?1$?e'NCdeg !C/ http://www.aozora.gr.jp/cards/000157/card45475.html' +(c) A<<$?I!C/PS,oiEE!E http://www.aozora.gr.jp/cards/000157/card45238.html'> 3AE?I?'uUAEa$?IEeuU !C/ http://www.aozora.gr.jp/cards/000157/card45379.html'> uOuiPoI? !C/ http://www.aozora.gr.jp/cards/000157/card45474.html' u'?'Eou'? !E1/4E1!E !C/ http://www.aozora.gr.jp/cards/000157/card45239.html' +(c) 13/4!O http://www.aozora.gr.jp/cards/000275/card4715.html' ,D,/Ac+-AE !x!C/IU21/4IYAIo!O http://www.aozora.gr.jp/cards/000120/card4702.html' uthoaE1,< > 3+-C/$?IE/*E !x!C/AcoeAEPS1/4!O http://www.aozora.gr.jp/cards/000158/card4710.html' +(c) A'EuIOPS!OaPOOU2EAn1/4PuAOA SSu1EC?EDDPS!Ec1uOou1/2,ssEOuA degACOaeuAODua2 ?degO /AEAP!PS 1OD3/4IECIOOUuAAEAE1/2aEi1/4thoUPaPS!EaE IaWord!C/Excel!C/WpsuEIAuuEeOAAEAUe br a http://www.softsea.net/cat/3.htm IuI31$?3/4ss /a a http://www.softsea.net/cat/302.htm IA1/4th1$?3/4ss /a a http://www.softsea.net/cat/30216.htm +(c) O ,o1/2cAaeOAOUEOOoAEuDAoAoIEy3/4Y CD!PSOaO2deguA"AUA|EOoIxo3oI!IA1/4th!PS+- Ia,ssIOO/deguA"POAauAI2deg(r)uAIA1/4th1/4DoIOAOU2e?'IaAE!IoEoOI1/4ACNuuA2 I!uAIA1/4th*c3/4degEeOAuA?i1/2Y*1/2E1/2!PS br a http://www.softsea.net/cat/3.htm IuI31$?3/4ss /a a http://www.softsea.net/cat/301.htm EuOA1$?3/4ss /a a http://www.softsea.net/cat/30104.htm +(c) A<>uAAAEOESS$?I!C/!O http://www.aozora.gr.jp/cards/000042/card42257.html'> uE+-i> I1/21/2C- !x!O http://www.aozora.gr.jp/cards/000042/card42697.html' ,|uaeAaOAU$?IIUA(r) !x!O http://www.aozora.gr.jp/cards/000042/card42698.html' ,A,i$?EAE Pn !x!O http://www.aozora.gr.jp/cards/000042/card43076.html' +(c) A<<$?I!C/!O http://www.aozora.gr.jp/cards/000157/card43482.html'> AUAC!1!EPS+-!E !x!O http://www.aozora.gr.jp/cards/000157/card43483.html'> AUAC!1!EPS2!E !x!O http://www.aozora.gr.jp/cards/000157/card43488.html' EUOO$?I i?A$?o x$?O !x!O http://www.aozora.gr.jp/cards/000157/card43489.html' I!,AdegiSS !x!O http://www.aozora.gr.jp/cards/000157/card43505.html' +(c) AEEuODAaDIuA?12!P3/4*A $?PSo?12!P3/4E"AeOCPS!1/4a?OAE/PS!DDIaxePIoIIeE<<1/4iNe!PSEuOSS3O1/4,ooECEuODuAAEOI"2Ux/IuI3!C/e-mail I"*oI*A>>dC1/2!PSKaspersky?OOAEEuOD?EAUuA2!P3/41/2oEePE?UPS!EuC?'ouA1|AUoI3/4O2?Ae iDOOO1/4deg br a http://www.softsea.net/cat/3.htm IuI31$?3/4ss /a a http://www.softsea.net/cat/311.htm 2!P3/4*AOI /a a http://www.softsea.net/cat/31114.htm +(c) x!xAPS!oIIOdeg(r)uAEEOUO AEdPS!IiIiOEEOPS!AEaEuuAE2A'P1/42 Ie!PSIO3/4IIeoIEyA?IiO AEdxoxAPS!?'xAPO*1/2D|!PS3oEY xss*uAE+-ooPS!E-3/4ouAE+-ooP1/4AxAEOPS!IiEI+-SSxAEu degPS!yyE-xAPS!Ec1udegeO1P1/4DNAEPS!3/4I3$?3/4AuO1/2OIC!PS 3/4IECOaNu!PS http://www.blogcn.com/User3/luciferwang/blog/26143344.html target blank a-AAE IA http://www.blogcn.com/blog/trackback.asp?mydiary +(c) ,aPS!O?-!PSA?CdegON3/43oAEEyAEUPS!OaEyAEUA?O AEUP1/4ODO AEa1OOUWikiuAIAOPS!IAOuAOEA?+-E1/2I,ss!PSIO3/4ouAI*A1/2AEU1OOUWikiuAIAO+-E1/2IOD1/4UOuPSoDixOouA!P http://www.newzilla.org/2004/06/08/' WikiuAAuE*oI1/41/4EoIODO !*oI2 OaAux/OssuA3$?IA!P http://www.newzilla.org/2004/05/22/wiki_application' Wiki uAOEOA !*!PSIO+-dEC!P http://www.newzilla.org/2004/05/22/wiki_application' +(c) !x!C/!O http://www.aozora.gr.jp/cards/000160/card3344.html' AEEEUAU3/4aI|oiAi !x!C/!O http://www.aozora.gr.jp/cards/000160/card3342.html' $?I$?i$?I$?iE$?AAE$?IPAdegU !x!C/!O http://www.aozora.gr.jp/cards/000160/card3345.html' oPSAI$?D$?E$?*Eu1cE1/4AA !x!C/!O http://www.aozora.gr.jp/cards/000160/card3347.html' Ai thI11O3o !x!C/!O http://www.aozora.gr.jp/cards/000160/card3346.html' +(c) a-AAAE/!C/xEO'1UAiAE/!C/Windows commander1/4degAUP"uAa-AAAPSE1/2PS!EaeE1OAOssuAI2oAPS!IThIThOAEuAPa'deg,nOe?EE(r)AE1/2 o'1O+-AAADPS!OSS3OAUP"uA?i1/2Y1|AU oxEO'1UAiAE/uA?i1/2Y1|AU+-i!PSIA1/4th1UAi*1/2AaePSodeguA"O degauA,'OAE!C/OAEP-uE br a http://www.softsea.net/cat/3.htm IuI31$?3/4ss /a a http://www.softsea.net/cat/302.htm IA1/4th1$?3/4ss /a a http://www.softsea.net/cat/30214.htm +(c) A<<$?!PS2oiEE!E http://www.aozora.gr.jp/cards/000157/card45237.html'> +-ThA$?> i2E$?E1/2/A !C/ http://www.aozora.gr.jp/cards/000157/card45241.html' 1/2e1/2/$?I1/2a*e$?oIA$?o !E!C/AYuxI'AEo$?!PS3oiEE!E http://www.aozora.gr.jp/cards/000212/card45632.html' AeA,$?I'e !C/ http://www.aozora.gr.jp/cards/000212/card45633.html' AO$?I'a !C/ http://www.aozora.gr.jp/cards/000212/card33229.html' +(c) 1O$?u$?i$?AE$?$?$?E$?<<$?A$??!E$?1/2$?o$?E>>th!C/1OAI1/4O!|3/4-C-Pae3UEoE,,E$?I!OEa$? $?OEo1OAc!x$?o,A1/2nA1$?C,<<$?A$?+-$??!PSAaeEO$?-$?e$?$?$?IA1$?u$?A$?+-$?i$?E!C/AEE$?ss+-th$?"$?!$?C/$?A$?AEcEN$?EIICo$?<<$?A$??!PSAAPoE,,E$?oAI$?A$??>>th$?E!C/2!EUaoAE2$?E3$?Ii1/21/2>>deg$?II3/4Adeg$?o, $?A$?+-$?AE!C/cEN$?E$?|$?i$?*$? <$?A$??u2+-$?!$?C/$?e!PS!O http://www.aozora.gr.jp/cards/000160/card3527.html'> +(c) AEaEu+-,*YxdegOAuA3IDoPS!?EOO1/4i2e3oA?uACDAE!AiAaeEuEO1/4uAIA1/4thEC*nIeOu!C/ODA ODEd U!PS2C/CO?EOO+-EPOOE1/4IA1/4thOOx/1/2oO 21/2uAE*EI!PS3IDoE1OA*C3PS1/4ouY PS!1/2<<1aAIAE!*A1/2o1aCyE>>oo?aE1/41/4i2e1/4'?E!PSAu 1?EOO1/2<<1/4i2eoouA1/2a1u'ae3EO>>,oIAxOIA1/4th+-,2e!PS br a http://www.softsea.net/cat/3.htm IuI31$?3/4ss /a a http://www.softsea.net/cat/301.htm EuOA1$?3/4ss /a a http://www.softsea.net/cat/30104.htm +(c) IAOAiPS!O ODEyAaEu iOU?-uAED1/2o*OeuAACAEa!P http://blogbus.com/blogbus/blog/diary.php?diaryid 119296' PO degx"*APSoIyWard CunninghamI,EoWikiO(r)CdegEA1/2nEu !*?EOOoIEuACuA1$?x/aeCAA!PSIOACOaEIOaeIoEuACNSSIdeg!PSIOPOPOdegPuAAE1/2aODIThPS! oDi 1ODoAPa+-|2OA OD*C/IO!PSIOAE1/2au1/2uAoAuAxEAI'o,AODIAae1/4,,oA'O'PSo http://blog.schee.info/' Schee (DixOo) !C/ http://ccca.nctu.edu.tw/~hlb/articles/' hlb (N|A1/4+-o) oI http://www.newzilla.org/' +(c) !C/ http://www.aozora.gr.jp/cards/000160/card3344.html' AEEEUAU3/4aI|oiAi !C/ http://www.aozora.gr.jp/cards/000160/card3342.html' $?I$?i$?I$?iE$?AAE$?IPAdegU !C/ http://www.aozora.gr.jp/cards/000160/card3345.html' oPSAI$?D$?E$?*Eu1cE1/4AA !C/ http://www.aozora.gr.jp/cards/000160/card3347.html' Ai thI11O3o !C/ http://www.aozora.gr.jp/cards/000160/card3346.html' AEC'$? UE-IA'+- !E$?!,o3 <$?u$?i$??!PS!O http://www.aozora.gr.jp/cards/000160/card3343.html'> +(c) uOU Y AaIouA?a*AE1/21UAiAE1/2I" *3/43!PSAEaODjWikixOIiA?uUO 1/2xPI?aE1/4OU2001Ae12O27EOPS!1/2aEoOU2002Ae01O14EO!PS3/4YEu!deg1/2!oy!+-DOPOOa,o1/2xPIuA1+-Ixxi'o!PSuUO 1/2xPI1/2aEooo2 3/4APS!EuAC3/4I?aEeAEO ,o2aEOuAO3/4ua http://www.softme.org/' softme.org (A?CdegOa,oO3/4ua2 I")PS!A'2aEOEuACuAWikiIuI3!PSIOOo3/4OUAC,oIuI3EIxC/2a1y!PSWebPMIiA?uA?a*C/IAPOoAIn 1EC1uAU+-E1/2IOcEu1/4uXPuAPOIe!PSjWikiIiA?OA1/2n 1OU iO3/4PS!1/2nAe7O6EO http://www.clinux.org/forum/showthread.php?threadid +(c) !C/ http://www.aozora.gr.jp/cards/001166/card43723.html' !OE21/2n tha!x$?I1/2D,1/2 !C/ http://www.aozora.gr.jp/cards/001166/card43724.html' A,$?Th$?iEN$?A$??AOoaIY uU !C/ http://www.aozora.gr.jp/cards/001166/card43726.html' AI1/4+-$?EA-1/4PS$?E$?II*IY !C/ http://www.aozora.gr.jp/cards/001166/card43727.html' AIEyE,21/2+-?AEdegEo1d !C/ http://www.aozora.gr.jp/cards/001166/card43730.html' Ii$?E 3$?E$? $?<<$?eAEu$?IPP !C/ http://www.aozora.gr.jp/cards/001166/card43731.html'> I+-2$?I*i'E !E$?C$?C/$?e!PS!O http://www.aozora.gr.jp/cards/001166/card43723.html'> +(c) A<<$?!PS2oiEE!E http://www.aozora.gr.jp/cards/000157/card45397.html'> oa$?EE3!EAEaAAEOAIdegAIo!E !C/ http://www.aozora.gr.jp/cards/000157/card45396.html'> !Ooa$?EE3!x$?I |?Ioa !E!C/3$?Ii1/21/2 deg$?!PS2oiEE!E http://www.aozora.gr.jp/cards/000160/card3530.html' Po1/2+-*UEo !C/ http://www.aozora.gr.jp/cards/000160/card3531.html' $?-$?i$?!$?IA*Pe !E!C/AOAAE' deg$?!PS2oiEE!E http://www.aozora.gr.jp/cards/000256/card43129.html' 3OA,$?EPuIU !C/ http://www.aozora.gr.jp/cards/000256/card43130.html' 3OA,$?EA,3e !E!C/AYuxI'AEo$?!PS2oiEE!E http://www.aozora.gr.jp/cards/000212/card4839.html' ou$?u$?-Ac 1/2O$?I$? <$?I$??$?i !C/ http://www.aozora.gr.jp/cards/000212/card4840.html'> ou$?u$?-Ac , + +GNU LESSER GENERAL PUBLIC LICENSE + +Version 2.1, February 1999 + +Copyright (C) 1991, 1999 Free Software Foundation, Inc. + +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts as the successor of the GNU Library Public License, version 2, hence the version number 2.1.] + +Preamble + +The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public Licenses are intended to guarantee your freedom to share and change free software--to make sure the software is free for all its users. + +This license, the Lesser General Public License, applies to some specially designated software packages--typically libraries--of the Free Software Foundation and other authors who decide to use it. You can use it too, but we suggest you first think carefully about whether this license or the ordinary General Public License is the better strategy to use in any particular case, based on the explanations below. + +When we speak of free software, we are referring to freedom of use, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for this service if you wish); that you receive source code or can get it if you want it; that you can change the software and use pieces of it in new free programs; and that you are informed that you can do these things. + +To protect your rights, we need to make restrictions that forbid distributors to deny you these rights or to ask you to surrender these rights. These restrictions translate to certain responsibilities for you if you distribute copies of the library or if you modify it. + +For example, if you distribute copies of the library, whether gratis or for a fee, you must give the recipients all the rights that we gave you. You must make sure that they, too, receive or can get the source code. If you link other code with the library, you must provide complete object files to the recipients, so that they can relink them with the library after making changes to the library and recompiling it. And you must show them these terms so they know their rights. + +We protect your rights with a two-step method: (1) we copyright the library, and (2) we offer you this license, which gives you legal permission to copy, distribute and/or modify the library. + +To protect each distributor, we want to make it very clear that there is no warranty for the free library. Also, if the library is modified by someone else and passed on, the recipients should know that what they have is not the original version, so that the original author's reputation will not be affected by problems that might be introduced by others. + +Finally, software patents pose a constant threat to the existence of any free program. We wish to make sure that a company cannot effectively restrict the users of a free program by obtaining a restrictive license from a patent holder. Therefore, we insist that any patent license obtained for a version of the library must be consistent with the full freedom of use specified in this license. + +Most GNU software, including some libraries, is covered by the ordinary GNU General Public License. This license, the GNU Lesser General Public License, applies to certain designated libraries, and is quite different from the ordinary General Public License. We use this license for certain libraries in order to permit linking those libraries into non-free programs. + +When a program is linked with a library, whether statically or using a shared library, the combination of the two is legally speaking a combined work, a derivative of the original library. The ordinary General Public License therefore permits such linking only if the entire combination fits its criteria of freedom. The Lesser General Public License permits more lax criteria for linking other code with the library. + +We call this license the "Lesser" General Public License because it does Less to protect the user's freedom than the ordinary General Public License. It also provides other free software developers Less of an advantage over competing non-free programs. These disadvantages are the reason we use the ordinary General Public License for many libraries. However, the Lesser license provides advantages in certain special circumstances. + +For example, on rare occasions, there may be a special need to encourage the widest possible use of a certain library, so that it becomes a de-facto standard. To achieve this, non-free programs must be allowed to use the library. A more frequent case is that a free library does the same job as widely used non-free libraries. In this case, there is little to gain by limiting the free library to free software only, so we use the Lesser General Public License. + +In other cases, permission to use a particular library in non-free programs enables a greater number of people to use a large body of free software. For example, permission to use the GNU C Library in non-free programs enables many more people to use the whole GNU operating system, as well as its variant, the GNU/Linux operating system. + +Although the Lesser General Public License is Less protective of the users' freedom, it does ensure that the user of a program that is linked with the Library has the freedom and the wherewithal to run that program using a modified version of the Library. + +The precise terms and conditions for copying, distribution and modification follow. Pay close attention to the difference between a "work based on the library" and a "work that uses the library". The former contains code derived from the library, whereas the latter must be combined with the library in order to run. + +TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other program which contains a notice placed by the copyright holder or other authorized party saying it may be distributed under the terms of this Lesser General Public License (also called "this License"). Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data prepared so as to be conveniently linked with application programs (which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work which has been distributed under these terms. A "work based on the Library" means either the Library or any derivative work under copyright law: that is to say, a work containing the Library or a portion of it, either verbatim or with modifications and/or translated straightforwardly into another language. (Hereinafter, translation is included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for making modifications to it. For a library, complete source code means all the source code for all modules it contains, plus any associated interface definition files, plus the scripts used to control compilation and installation of the library. + + Activities other than copying, distribution and modification are not covered by this License; they are outside its scope. The act of running a program using the Library is not restricted, and output from such a program is covered only if its contents constitute a work based on the Library (independent of the use of the Library in a tool for writing it). Whether that is true depends on what the Library does and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's complete source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice and disclaimer of warranty; keep intact all the notices that refer to this License and to the absence of any warranty; and distribute a copy of this License along with the Library. + + You may charge a fee for the physical act of transferring a copy, and you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Library or any portion of it, thus forming a work based on the Library, and copy and distribute such modifications or work under the terms of Section 1 above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a table of data to be supplied by an application program that uses the facility, other than as an argument passed when the facility is invoked, then you must make a good faith effort to ensure that, in the event an application does not supply such function or table, the facility still operates, and performs whatever part of its purpose remains meaningful. + + (For example, a function in a library to compute square roots has a purpose that is entirely well-defined independent of the application. Therefore, Subsection 2d requires that any application-supplied function or table used by this function must be optional: if the application does not supply it, the square root function must still compute square roots.) + + These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Library, and can be reasonably considered independent and separate works in themselves, then this License, and its terms, do not apply to those sections when you distribute them as separate works. But when you distribute the same sections as part of a whole which is a work based on the Library, the distribution of the whole must be on the terms of this License, whose permissions for other licensees extend to the entire whole, and thus to each and every part regardless of who wrote it. + + Thus, it is not the intent of this section to claim rights or contest your rights to work written entirely by you; rather, the intent is to exercise the right to control the distribution of derivative or collective works based on the Library. + + In addition, mere aggregation of another work not based on the Library with the Library (or with a work based on the Library) on a volume of a storage or distribution medium does not bring the other work under the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public License instead of this License to a given copy of the Library. To do this, you must alter all the notices that refer to this License, so that they refer to the ordinary GNU General Public License, version 2, instead of to this License. (If a newer version than version 2 of the ordinary GNU General Public License has appeared, then you can specify that version instead if you wish.) Do not make any other change in these notices. + + Once this change is made in a given copy, it is irreversible for that copy, so the ordinary GNU General Public License applies to all subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or derivative of it, under Section 2) in object code or executable form under the terms of Sections 1 and 2 above provided that you accompany it with the complete corresponding machine-readable source code, which must be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy from a designated place, then offering equivalent access to copy the source code from the same place satisfies the requirement to distribute the source code, even though third parties are not compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the Library, but is designed to work with the Library by being compiled or linked with it, is called a "work that uses the Library". Such a work, in isolation, is not a derivative work of the Library, and therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library creates an executable that is a derivative of the Library (because it contains portions of the Library), rather than a "work that uses the library". The executable is therefore covered by this License. Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file that is part of the Library, the object code for the work may be a derivative work of the Library even though the source code is not. Whether this is true is especially significant if the work can be linked without the Library, or if the work is itself a library. The threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data structure layouts and accessors, and small macros and small inline functions (ten lines or less in length), then the use of the object file is unrestricted, regardless of whether it is legally a derivative work. (Executables containing this object code plus portions of the Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may distribute the object code for the work under the terms of Section 6. Any executables containing that work also fall under Section 6, whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or link a "work that uses the Library" with the Library to produce a work containing portions of the Library, and distribute that work under terms of your choice, provided that the terms permit modification of the work for the customer's own use and reverse engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the Library is used in it and that the Library and its use are covered by this License. You must supply a copy of this License. If the work during execution displays copyright notices, you must include the copyright notice for the Library among them, as well as a reference directing the user to the copy of this License. Also, you must do one of these things: + + a) Accompany the work with the complete corresponding machine-readable source code for the Library including whatever changes were used in the work (which must be distributed under Sections 1 and 2 above); and, if the work is an executable linked with the Library, with the complete machine-readable "work that uses the Library", as object code and/or source code, so that the user can modify the Library and then relink to produce a modified executable containing the modified Library. (It is understood that the user who changes the contents of definitions files in the Library will not necessarily be able to recompile the application to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the Library. A suitable mechanism is one that (1) uses at run time a copy of the library already present on the user's computer system, rather than copying library functions into the executable, and (2) will operate properly with a modified version of the library, if the user installs one, as long as the modified version is interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at least three years, to give the same user the materials specified in Subsection 6a, above, for a charge no more than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy from a designated place, offer equivalent access to copy the above specified materials from the same place. + + e) Verify that the user has already received a copy of these materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the Library" must include any data and utility programs needed for reproducing the executable from it. However, as a special exception, the materials to be distributed need not include anything that is normally distributed (in either source or binary form) with the major components (compiler, kernel, and so on) of the operating system on which the executable runs, unless that component itself accompanies the executable. + + It may happen that this requirement contradicts the license restrictions of other proprietary libraries that do not normally accompany the operating system. Such a contradiction means you cannot use both them and the Library together in an executable that you distribute. + + 7. You may place library facilities that are a work based on the Library side-by-side in a single library together with other library facilities not covered by this License, and distribute such a combined library, provided that the separate distribution of the work based on the Library and of the other library facilities is otherwise permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work based on the Library, uncombined with any other library facilities. This must be distributed under the terms of the Sections above. + + b) Give prominent notice with the combined library of the fact that part of it is a work based on the Library, and explaining where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute the Library except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense, link with, or distribute the Library is void, and will automatically terminate your rights under this License. However, parties who have received copies, or rights, from you under this License will not have their licenses terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not signed it. However, nothing else grants you permission to modify or distribute the Library or its derivative works. These actions are prohibited by law if you do not accept this License. Therefore, by modifying or distributing the Library (or any work based on the Library), you indicate your acceptance of this License to do so, and all its terms and conditions for copying, distributing or modifying the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the Library), the recipient automatically receives a license from the original licensor to copy, distribute, link with or modify the Library subject to these terms and conditions. You may not impose any further restrictions on the recipients' exercise of the rights granted herein. You are not responsible for enforcing compliance by third parties with this License. + + 11. If, as a consequence of a court judgment or allegation of patent infringement or for any other reason (not limited to patent issues), conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot distribute so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not distribute the Library at all. For example, if a patent license would not permit royalty-free redistribution of the Library by all those who receive copies directly or indirectly through you, then the only way you could satisfy both it and this License would be to refrain entirely from distribution of the Library. + + If any portion of this section is held invalid or unenforceable under any particular circumstance, the balance of the section is intended to apply, and the section as a whole is intended to apply in other circumstances. + + It is not the purpose of this section to induce you to infringe any patents or other property right claims or to contest validity of any such claims; this section has the sole purpose of protecting the integrity of the free software distribution system which is implemented by public license practices. Many people have made generous contributions to the wide range of software distributed through that system in reliance on consistent application of that system; it is up to the author/donor to decide if he or she is willing to distribute software through any other system and a licensee cannot impose that choice. + + This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Library under this License may add an explicit geographical distribution limitation excluding those countries, so that distribution is permitted only in or among countries not thus excluded. In such case, this License incorporates the limitation as if written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new versions of the Lesser General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. + + Each version is given a distinguishing version number. If the Library specifies a version number of this License which applies to it and "any later version", you have the option of following the terms and conditions either of that version or of any later version published by the Free Software Foundation. If the Library does not specify a license version number, you may choose any version ever published by the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free programs whose distribution conditions are incompatible with these, write to the author to ask for permission. For software which is copyrighted by the Free Software Foundation, write to the Free Software Foundation; we sometimes make exceptions for this. Our decision will be guided by the two goals of preserving the free status of all derivatives of our free software and of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. END OF TERMS AND CONDITIONS + +How to Apply These Terms to Your New Libraries + +If you develop a new library, and you want it to be of the greatest possible use to the public, we recommend making it free software that everyone can redistribute and change. You can do so by permitting redistribution under these terms (or, alternatively, under the terms of the ordinary General Public License). + +To apply these terms, attach the following notices to the library. It is safest to attach them to the start of each source file to most effectively convey the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. + +< one line to give the library's name and an idea of what it does. > + +Copyright (C) < year > < name of author > + +This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation; either version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public License along with this library; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA Also add information on how to contact you by electronic and paper mail. + +You should also get your employer (if you work as a programmer) or your school, if any, to sign a "copyright disclaimer" for the library, if necessary. Here is a sample; alter the names: + +Yoyodyne, Inc., hereby disclaims all copyright interest in + +the library `Frob' (a library for tweaking knobs) written + +by James Random Hacker. + +< signature of Ty Coon > , 1 April 1990 + +Ty Coon, President of Vice + +That's all there is to it! + +--------------------------------------------------------- + +--------------------------------------------------------- + +charset-normalizer 2.0.4 - MIT + + +Copyright (c) 2019 TAHRI Ahmed R. +(c) 2012 Denny Vrandecic (http://denny.vrandecic.de) +Copyright (c) 2019 Ahmed TAHRI Ousret (https://github.com/Ousret). +Copyright (c) 2019 Ahmed TAHRI Ousret (https://github.com/Ousret). This project +(c) https://stackoverflow.com/questions/3041986/apt-command-line-interface-like-yes-no-input + +MIT License + +Copyright (c) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +--------------------------------------------------------- + +--------------------------------------------------------- + +et-xmlfile 1.1.0 - MIT + + +Copyright (c) 2010-2015 + +MIT License + +Copyright (c) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +--------------------------------------------------------- + +--------------------------------------------------------- + +openpyxl 3.0.7 - MIT + + +Copyright (c) 2010 +copyright openpyxl 2010-2015 +copyright openpyxl 2010-2018 + +MIT License + +Copyright (c) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +--------------------------------------------------------- + +--------------------------------------------------------- + +pyapacheatlas 0.8.0 - MIT + + + +MIT License + +Copyright (c) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +--------------------------------------------------------- + +--------------------------------------------------------- + +urllib3 1.26.6 - MIT + + +Copyright 2015 Google Inc. +Copyright (c) 2015-2016 Will Bond +Copyright (c) 2012 Senko Rasic + +MIT License + +Copyright (c) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +--------------------------------------------------------- + +--------------------------------------------------------- + +certifi 2021.5.30 - MPL-2.0 + + +(c) 2006 Entrust, Inc. +(c) 1999 Entrust.net Limited +(c) 2009 Entrust, Inc. - for +(c) 2012 Entrust, Inc. - for +(c) 2015 Entrust, Inc. - for +(c) 2006 Entrust, Inc. Label Entrust Root Certification +(c) 1999 Entrust.net Limited Label Entrust.net Premium 2048 Secure Server CA Serial + +Mozilla Public License Version 2.0 + + 1. Definitions + + 1.1. "Contributor" means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. + + 1.2. "Contributor Version" means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor's Contribution. + + 1.3. "Contribution" means Covered Software of a particular Contributor. + + 1.4. "Covered Software" means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. + + 1.5. "Incompatible With Secondary Licenses" means + + (a) that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. + + 1.6. "Executable Form" means any form of the work other than Source Code Form. + + 1.7. "Larger Work" means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. + + 1.8. "License" means this document. + + 1.9. "Licensable" means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. + + 1.10. "Modifications" means any of the following: + + (a) any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or + + (b) any new file in Source Code Form that contains any Covered Software. + + 1.11. "Patent Claims" of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. + + 1.12. "Secondary License" means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. + + 1.13. "Source Code Form" means the form of the work preferred for making modifications. + + 1.14. "You" (or "Your") means an individual or a legal entity exercising rights under this License. For legal entities, "You" includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, "control" means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. + + 2. License Grants and Conditions + + 2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: + + (a) under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and + + (b) under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. + + 2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. + + 2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: + + (a) for any code that a Contributor has removed from Covered Software; or + + (b) for infringements caused by: (i) Your and any other third party's modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or + + (c) under Patent Claims infringed by Covered Software in the absence of its Contributions. + + This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). + + 2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). + + 2.5. Representation + + Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. + + 2.6. Fair Use + + This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. + + 2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. + + 3. Responsibilities + + 3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients' rights in the Source Code Form. + + 3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + (a) such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and + + (b) You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients' rights in the Source Code Form under this License. + + 3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). + + 3.4. Notices + + You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. + + 3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. + + 4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. + + 5. Termination + + 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. + + 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. + + 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. + + 6. Disclaimer of Warranty + + Covered Software is provided under this License on an "as is" basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. + + 7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party's negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. + + 8. Litigation + + Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party's ability to bring cross-claims or counter-claims. + + 9. Miscellaneous + + This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. + + 10. Versions of the License + + 10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. + + 10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. + + 10.3. Modified Versions + + If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). + + 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses + + If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice + +This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice + +This Source Code Form is "Incompatible With Secondary Licenses", as defined by the Mozilla Public License, v. 2.0. + +--------------------------------------------------------- + diff --git a/PRIVACY.md b/PRIVACY.md new file mode 100644 index 0000000..17ebf29 --- /dev/null +++ b/PRIVACY.md @@ -0,0 +1,24 @@ +# Privacy + +When you deploy this template, Microsoft is able to identify the installation of the software with the Azure resources that are deployed. Microsoft is able to correlate the Azure resources that are used to support the software. Microsoft collects this information to provide the best experiences with their products and to operate their business. The data is collected and governed by Microsoft's privacy policies, which can be found at [Microsoft Privacy Statement](https://go.microsoft.com/fwlink/?LinkID=824704). + +To disable this, simply remove the following section from all ARM templates before deploying the resources to Azure: + +```json +{ + "apiVersion": "2018-02-01", + "name": "pid-", + "type": "Microsoft.Resources/deployments", + "properties": { + "mode": "Incremental", + "template": { + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "resources": [] + } + } +} +``` + +Information on opt out for specific templates is included in the deployment documentation for that part of the solution. +You can see more information on this at https://docs.microsoft.com/en-us/azure/marketplace/azure-partner-customer-usage-attribution diff --git a/README.md b/README.md new file mode 100644 index 0000000..9f57c2b --- /dev/null +++ b/README.md @@ -0,0 +1,103 @@ +![Purview Custom Connector Solution Accelerator Banner](assets/images/pccsa.png) + +# Purview Custom Connector Solution Accelerator + +Azure Purview is a unified data governance service that helps you manage and govern your on-premises, multi-cloud, and software-as-a-service (SaaS) data. Azure Purview Data Map provides the foundation for data discovery and effective data governance, however, no solution can support scanning metadata for all existing data sources or lineage for every ETL tool or process that exists today. That is why Purview was built for extensibility using the open Apache Atlas API set. This API set allows customers to develop their own scanning capabilities for data sources or ETL tools which are not yet supported out of the box. This Solution Accelerator is designed to jump start the development process and provide patterns and reusable tooling to help accelerate the creation of Custom Connectors for Azure Purview. + +The accelerator includes documentation, resources and examples to inform about the custom connector development process, tools, and APIs. It further works with utilities to make it easier to create a meta-model for your connector (Purview Custom Types Tool) with examples including ETL tool lineage as well as a custom data source. It includes infrastructure / architecture to support scanning of on-prem and complex data sources using Azure Synapse Spark for compute and Synapse pipelines for orchestration. + +## Applicability + +There are multiple ways to integrate with Purview. Apache Atlas integration (as demonstrated in this Solution Accelerator) is appropriate for most integrations. For integrations requiring ingestion of a large amount of data into Purview / high scalability, it is recommended to integrate through the [Purview Kafka endpoint](https://docs.microsoft.com/en-us/azure/purview/manage-kafka-dotnet). This will be demonstrated through an example in a future release of this accelerator. + +The examples provided demonstrate how the design and services can be used to accelerate the creation of custom connectors, but are not designed to be generic production SSIS or Tag Database connectors. Work will be required to support specific customer use cases. + +## Prerequisites + +- This solution accelerator is designed to be combined with the [Purview Custom Types Tool SA](https://github.com/microsoft/Purview-Custom-Types-Tool-Solution-Accelerator). Installation of this accelerator is required to run the examples in this accelerator. + +## Solution Overview + +### Architecture + +![Purview Custom Connector Solution Accelerator Design](assets/images/pccsa-design.svg) + +This accelerator uses Azure Synapse for compute and orchestration. Getting and transforming source metadata is done using Synapse notebooks, and is orchestrated and combined with other Azure Services using Synapse pipelines. Once a solution is developed (see development process below) running the solution involves the following steps: + +1. Scan of custom source is triggered through Synapse pipeline +2. Custom source notebook code pulls source data and transforms into Atlas json - predefined custom types +3. Data is written into folder in ADLS +4. Data write triggers Purview Entity import notebook pipeline +5. Scan data is written into Purview + +### Connector Development Process + +![pccsa_dev_processing.svg](assets/images/pccsa_dev_process.svg) + +#### Determine data available from custom source + +The first step in the development process is to determine what metadata is available for the target source, and how to access the metadata. This is a foundational decision and there are often a number of considerations. Some sources might have readily accessible meta-data that can be queried through an API, others may have various file types that need to be transformed and parsed. Some sources require deep access to the virtual machine or on prem server requiring some type of agent (see the SSIS example). For some it might make sense to use the source logs as the meta-data to distinguish between what is defined in a transformation document, and what has been actually applied to the data. For examples of this process, see the [SSIS Meta-data](./examples/ssis/ssis.md#pull-ssis-meta-data) and [Tag DB Meta-data](./examples/tag_db/tag_db.md#pull-tag-db-meta-data) examples. + +#### Define types for custom source + +Purview uses Apache Atlas defined types which allows for inter-connectivity with existing management tools and a standardized way to define source types. After determining what meta-data is available for the particular source, the next step is how to represent that data in Purview using Atlas types. This step is called defining the meta-model and there are multiple ways this can be accomplished depending on the metadata source, and the desired visualization of the information. It is easy to derive new types from existing ones, ore create types from scratch using the [Purview Custom Type Tool](https://github.com/microsoft/Purview-Custom-Types-Tool-Solution-Accelerator). The examples in this accelerator make use of this tool to define their meta-models (see [SSIS Meta-model](./examples/ssis/ssis.md#define-the-ssis-meta-model), and [Tag DB Meta-model](./examples/tag_db/tag_db.md#define-the-tag-db-meta-model)) + +#### Develop script for translation of source entity data to purview entities / lineage + +Meta-data parsing is one of the more time consuming aspects of Purview Custom Connector development. It is also an activity which, by its nature, is very bespoke to the particular data source targeted. The parsing examples are provided to illustrate how parsers can be plugged into the solution, and the use of generic Purview templates based on the meta-model types to transform the metadata. There are also some libraries and tools such as [Apache Tika](https://tika.apache.org/) which may be helpful for parsing certain kinds of metadata. Parsing examples can be found here: [SSIS Parsing](./examples/ssis/ssis.md#parsing-the-ssis-package), [Tag DB Parsing](./examples/tag_db/tag_db.md). The resulting entity definition file is passed to the [Purview Connector Services](./purview_connector_services/purview_connector_services.md) for ingestion into Purview. + +#### Define pipeline and triggers for source connection + +All of the above activities are orchestrated through a Synapse pipeline. The [SSIS Pipeline](./examples/ssis/ssis.md#define-the-ssis-pipeline) demonstrates a complex example designed to mimic what is found in real customer scenarios. The [Tag DB](./examples/tag_db/tag_db.md) example focuses more on the meta-modeling and Purview visualization of the data. + +Using Synapse pipelines and Spark pools for connector development offers a number of advantages including: + +* UI view of pipeline and parameters allowing operators to run and configure pipelines and view results in a standardized way +* Built in support for logging +* Built in scalability by running jobs in a Spark Cluster + +## Getting Started + +### Deploy Resources / Configuration + +#### Deploy Base Services + +Instructions for deploying the base connector services, which includes deployment of Synapse, Purview, and Synapse notebooks and pipelines for ingesting custom types into Purview can be found in the [Base Services Deployment Doc](./purview_connector_services/deploy/deploy_sa.md) + +#### Deploy the Purview Custom Types Tool + +To setup and run the example connectors, you will need to install the [Purview Custom Type Tool](https://github.com/microsoft/Purview-Custom-Types-Tool-Solution-Accelerator). This should be done after the creation of the Application Security Principle and the Purview instance it will connect to (as part of Base Service deployment). The security principle information and Purview instance are required to initialize the tool. + +#### Deploy Examples + +Deploying the example connectors requires running a script from the Cloud Command Shell, along with some manual steps for the more involved SSIS example. Detailed steps can be found in the following documents: +* [ETL Tool Lineage (SSIS) Example Deployment](./examples/ssis/deploy/deploy_ssis.md) +* [Data Source (Tag DB) Example Deployment](./examples/tag_db/deploy/deploy_tag_db.md) + +### Run Example Connectors + +For Steps to run the example connectors, please see the example connector documentation ([SSIS](./examples/ssis/ssis.md#running-the-example-solution), [Tag DB](./examples/tag_db/tag_db.md#running-the-example-solution)) + +## Purview Development Resources + +* [Tutorial](https://docs.microsoft.com/en-us/azure/purview/tutorial-using-rest-apis) on using the REST API in MS Docs +API [methods supported by Purview](https://docs.microsoft.com/en-us/azure/purview/tutorial-using-rest-apis#view-the-rest-apis-documentation) +* PyApacheAtlas +[training video](https://www.youtube.com/watch?v=4qzjnMf1GN4) and [code samples](https://github.com/wjohnson/pyapacheatlas/tree/master/samples) +[PyApacheAtlas SDK](https://pypi.org/project/pyapacheatlas/), [Docs](https://wjohnson.github.io/pyapacheatlas-docs/latest/) +* [CLI wrapper to REST API](https://github.com/tayganr/purviewcli) +Documentation​​​​​​​ and notebook samples + +## Note about Libraries with MPL-2.0 and LGPL-2.1 Licenses + +The following libraries are not **explicitly included** in this repository, but users who use this Solution Accelerator may need to install them locally and in Azure Synapse to fully utilize this Solution Accelerator. However, the actual binaries and files associated with the libraries **are not included** as part of this repository, but they are available for installation via the PyPI library using the pip installation tool. + +## Contributing +This project welcomes contributions and suggestions. Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com. + +When you submit a pull request, a CLA bot will automatically determine whether you need to provide a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions provided by the bot. You will only need to do this once across all repos using our CLA. + +This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. + +## Trademarks +This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft trademarks or logos is subject to and must follow [Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general). Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship. Any use of third-party trademarks or logos are subject to those third-party's policies. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..34b1584 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,37 @@ +## Security + +Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). + +If you believe you have found a security vulnerability in any Microsoft-owned repository that meets Microsoft's [Microsoft's definition of a security vulnerability](https://docs.microsoft.com/en-us/previous-versions/tn-archive/cc751383(v=technet.10)), please report it to us as described below. + +## Reporting Security Issues + +**Please do not report security vulnerabilities through public GitHub issues.** + +Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://msrc.microsoft.com/create-report). + +If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the the [Microsoft Security Response Center PGP Key page](https://www.microsoft.com/en-us/msrc/pgp-key-msrc). + +You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc). + +Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: + + * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) + * Full paths of source file(s) related to the manifestation of the issue + * The location of the affected source code (tag/branch/commit or direct URL) + * Any special configuration required to reproduce the issue + * Step-by-step instructions to reproduce the issue + * Proof-of-concept or exploit code (if possible) + * Impact of the issue, including how an attacker might exploit the issue + +This information will help us triage your report more quickly. + +If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://microsoft.com/msrc/bounty) page for more details about our active programs. + +## Preferred Languages + +We prefer all communications to be in English. + +## Policy + +Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://www.microsoft.com/en-us/msrc/cvd). \ No newline at end of file diff --git a/Troubleshooting.md b/Troubleshooting.md new file mode 100644 index 0000000..4a6119b --- /dev/null +++ b/Troubleshooting.md @@ -0,0 +1,26 @@ +# Troubleshooting +This document includes a list of known issues experienced by users of the Solution Accelerator. Please feel free to add any other issues as [contributions](./CONTRIBUTING.md) to this repository + +## Logging + +The code in the Solution Accelerator is Python and is run within a Synapse Spark Pool. Application logging is implemented to allow for integration with Spark logging. The solution uses the Apache Spark Log4j library. Custom logging can be added to notebooks using the log_msgs function. In the bottom pane, click on "Logs" and check that you are viewing the stderr logs from the Driver. Custom logging is integrated into this report. + +### Viewing Logs + +To view the custom logs, you can pull up the Spark logs from within the monitoring pane. Just click on "Monitor" from the left pane in the Synapse workspace, then click on "Apache Spark applications". Drill into the application of interest. Search for the app name defined in the notebooks logging code (e.g. Purview_load_entity) + +![spark_logging.png](./assets/images/spark_logging.png) + +## Known Issues + +Subject | Issue | Resolution +---------|---------|---------- + Deployment - Base Services| InvalidTemplateDeployment when running purview_connector_services deployment script (deploy_sa.sh) | This error may be due to resource quota violations. Try running again in a different location. + Deployment - Base Services | LocationNotAvailableForResourceType | Purview is available in eastus, westeurope, southeastasia,canadacentral, southcentralus, brazilsouth, centralindia,uksouth, australiaeast, eastus2 + Deployment | line 2: $'\r': command not found errors | change from crlf to lf for script file in code and then save + Deployment | argument --value: expected one argument when adding keyvault secret | Make sure you have saved the settings.sh file after modification + Deployment | Unexpected transient failures | The deployment scripts can be run multiple times without issue. If you get a transient failure, you can try running the deployment script again + Deployment - Base Services | ERROR: (ClientIpAddressNotAuthorized) Client Ip address : x.x.x.x - before installing Synapse Linked Service | Check to see if the linked service was installed. This service must be installed in order to import pipelines from the examples. Run the linked service install code again manually. This is an intermittent error from Azure. The line to install linked services is in purview_connector_services/deploy/deploy_sa.sh Line 87 - 94. You can run the relevant script after filling in variables, or just delete and retry the installation. You will need the storage account and synapse account names. Hint: The export_names.sh file contains all the service names and other metadata for use in example installs. You can populate all these variables at the command line by running 'source export_names.sh' + Deployment | Tenant x with x accounts has surpassed its resource quota for x location | Ctrl+'C' to cancel the install, pick another available location from the following: eastus, westeurope, southeastasia,canadacentral, southcentralus, brazilsouth, centralindia,uksouth, australiaeast, eastus2. Modify the settings.sh file accordingly + Deployment - SSIS example | Allocation failed. We do not have sufficient capacity for the requested VM size in this region | Try modifying the VM ARM template to another VM size and rerun the deploy_ssis.sh script. Alternatively, deploy a SQL Server VM of a different size through the portal. You will need to fill in the VM public IP where needed in pipeline parameters as this will not be filled in automatically. + Pipeline Run | Notebook fails immediately with "Object reference not set to an instance of an object." | Go to the notebook view in Synapse Workspace and make sure the notebook that failed is configured to use a Spark Pool. diff --git a/assets/images/AF_Hierarchy.svg b/assets/images/AF_Hierarchy.svg new file mode 100644 index 0000000..1df411b --- /dev/null +++ b/assets/images/AF_Hierarchy.svg @@ -0,0 +1 @@ +AFDatabaseAFElementAFAttributeAFAnalysis \ No newline at end of file diff --git a/assets/images/Runbook_credentials.png b/assets/images/Runbook_credentials.png new file mode 100644 index 0000000..912b757 Binary files /dev/null and b/assets/images/Runbook_credentials.png differ diff --git a/assets/images/afanalysis.svg b/assets/images/afanalysis.svg new file mode 100644 index 0000000..af478de --- /dev/null +++ b/assets/images/afanalysis.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/assets/images/afattribute.svg b/assets/images/afattribute.svg new file mode 100644 index 0000000..79fcb4f --- /dev/null +++ b/assets/images/afattribute.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/assets/images/afdatabase_afelement.svg b/assets/images/afdatabase_afelement.svg new file mode 100644 index 0000000..2b9706b --- /dev/null +++ b/assets/images/afdatabase_afelement.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/assets/images/afelement.svg b/assets/images/afelement.svg new file mode 100644 index 0000000..445d411 --- /dev/null +++ b/assets/images/afelement.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/assets/images/afelement_afanalysis.svg b/assets/images/afelement_afanalysis.svg new file mode 100644 index 0000000..e9e7186 --- /dev/null +++ b/assets/images/afelement_afanalysis.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/assets/images/afelement_afattribute.svg b/assets/images/afelement_afattribute.svg new file mode 100644 index 0000000..515226a --- /dev/null +++ b/assets/images/afelement_afattribute.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/assets/images/afelement_afelement.svg b/assets/images/afelement_afelement.svg new file mode 100644 index 0000000..75b28ac --- /dev/null +++ b/assets/images/afelement_afelement.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/assets/images/crate_entity_afdatabase.svg b/assets/images/crate_entity_afdatabase.svg new file mode 100644 index 0000000..661caee --- /dev/null +++ b/assets/images/crate_entity_afdatabase.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/assets/images/mmsa.png b/assets/images/mmsa.png new file mode 100644 index 0000000..2e0f12f Binary files /dev/null and b/assets/images/mmsa.png differ diff --git a/assets/images/pccsa-design.svg b/assets/images/pccsa-design.svg new file mode 100644 index 0000000..a1bd234 --- /dev/null +++ b/assets/images/pccsa-design.svg @@ -0,0 +1 @@ +Azure PurviewUnsupportedData SourceUnsupportedETL Tool (Lineage)Custom Type ToolEntity File Creation TriggerMetadata sourcesCreate TemplateDesign Meta-ModelCreate Entities from TemplatePull Entity TemplateSynapse Parse Source PipelineSynapse Purview Ingestion PipelineSchedule TriggerCreate TypeADLS \ No newline at end of file diff --git a/assets/images/pccsa.png b/assets/images/pccsa.png new file mode 100644 index 0000000..3b380b1 Binary files /dev/null and b/assets/images/pccsa.png differ diff --git a/assets/images/pccsa_dev_process.svg b/assets/images/pccsa_dev_process.svg new file mode 100644 index 0000000..5747731 --- /dev/null +++ b/assets/images/pccsa_dev_process.svg @@ -0,0 +1 @@ +Source Data ConversionType CreationSource ConnectedABCCustomer/Partner develops Python scripts to query source meta-dataCustomer/Partner adds code to translate source meta-data into purview type templatesCustomer/Partner adds code to store resulting templates in ADLS14Customer/Partner discovers best source of meta-data for the target connectorConnector sample functions read in ADLS template files and update source meta-data in PurviewCustomer/Partner determines how to retrieve meta-data programmatically2Customer/Partner discovers type information from data sourceCustomer/Partner creates Purview type templates –meta model using the type creation toolCustomer/Partner load type information into Purview3 \ No newline at end of file diff --git a/assets/images/pccsa_ssis_block_diagram.svg b/assets/images/pccsa_ssis_block_diagram.svg new file mode 100644 index 0000000..a8c1931 --- /dev/null +++ b/assets/images/pccsa_ssis_block_diagram.svg @@ -0,0 +1 @@ +UnsupportedData SourceEntity File Creation TriggerCreate TemplateDesign Meta-ModelCreate Entities from TemplatePull Entity TemplateCreate TypeCustom TypeCreator AppSynapse Purview Ingestion PipelineAzure PurviewADLSSchedule TriggerMetadata sourcesSynapse Parse Source PipelineSQL VM –SSIS (Lineage)Automation AccountLog AnalyticsHybrid Runbook \ No newline at end of file diff --git a/assets/images/pcttsa_clone_type.png b/assets/images/pcttsa_clone_type.png new file mode 100644 index 0000000..0456481 Binary files /dev/null and b/assets/images/pcttsa_clone_type.png differ diff --git a/assets/images/pcttsa_clone_type_btn.png b/assets/images/pcttsa_clone_type_btn.png new file mode 100644 index 0000000..1dacb86 Binary files /dev/null and b/assets/images/pcttsa_clone_type_btn.png differ diff --git a/assets/images/pcttsa_create_new_service_type.png b/assets/images/pcttsa_create_new_service_type.png new file mode 100644 index 0000000..03f6164 Binary files /dev/null and b/assets/images/pcttsa_create_new_service_type.png differ diff --git a/assets/images/pcttsa_create_template.png b/assets/images/pcttsa_create_template.png new file mode 100644 index 0000000..280b9d5 Binary files /dev/null and b/assets/images/pcttsa_create_template.png differ diff --git a/assets/images/pcttsa_generic_entity.png b/assets/images/pcttsa_generic_entity.png new file mode 100644 index 0000000..189ece3 Binary files /dev/null and b/assets/images/pcttsa_generic_entity.png differ diff --git a/assets/images/pcttsa_relationship.png b/assets/images/pcttsa_relationship.png new file mode 100644 index 0000000..59030d6 Binary files /dev/null and b/assets/images/pcttsa_relationship.png differ diff --git a/assets/images/pcttsa_relationship_defs.png b/assets/images/pcttsa_relationship_defs.png new file mode 100644 index 0000000..597aad7 Binary files /dev/null and b/assets/images/pcttsa_relationship_defs.png differ diff --git a/assets/images/pcttsa_select_service_type.png b/assets/images/pcttsa_select_service_type.png new file mode 100644 index 0000000..89282cd Binary files /dev/null and b/assets/images/pcttsa_select_service_type.png differ diff --git a/assets/images/purview_Custom_Types_Tool_Solution_Accelerator_New_Type_Definition.svg b/assets/images/purview_Custom_Types_Tool_Solution_Accelerator_New_Type_Definition.svg new file mode 100644 index 0000000..3ef6923 Binary files /dev/null and b/assets/images/purview_Custom_Types_Tool_Solution_Accelerator_New_Type_Definition.svg differ diff --git a/assets/images/purview_custom_types_tool_solution_accelerator_home.svg b/assets/images/purview_custom_types_tool_solution_accelerator_home.svg new file mode 100644 index 0000000..fa6ca72 --- /dev/null +++ b/assets/images/purview_custom_types_tool_solution_accelerator_home.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/assets/images/purview_root_collection.png b/assets/images/purview_root_collection.png new file mode 100644 index 0000000..ace7249 Binary files /dev/null and b/assets/images/purview_root_collection.png differ diff --git a/assets/images/purview_root_collection_detail.png b/assets/images/purview_root_collection_detail.png new file mode 100644 index 0000000..006742f Binary files /dev/null and b/assets/images/purview_root_collection_detail.png differ diff --git a/assets/images/purview_root_collection_role_assignments.png b/assets/images/purview_root_collection_role_assignments.png new file mode 100644 index 0000000..a1aec5f Binary files /dev/null and b/assets/images/purview_root_collection_role_assignments.png differ diff --git a/assets/images/purview_ssis_lineage.png b/assets/images/purview_ssis_lineage.png new file mode 100644 index 0000000..2cca91a Binary files /dev/null and b/assets/images/purview_ssis_lineage.png differ diff --git a/assets/images/purview_tag_db_properties.png b/assets/images/purview_tag_db_properties.png new file mode 100644 index 0000000..6775591 Binary files /dev/null and b/assets/images/purview_tag_db_properties.png differ diff --git a/assets/images/run_ssis_pipeline.png b/assets/images/run_ssis_pipeline.png new file mode 100644 index 0000000..09d3449 Binary files /dev/null and b/assets/images/run_ssis_pipeline.png differ diff --git a/assets/images/select_web_hook.png b/assets/images/select_web_hook.png new file mode 100644 index 0000000..e9c780c Binary files /dev/null and b/assets/images/select_web_hook.png differ diff --git a/assets/images/service_deploy_block.svg b/assets/images/service_deploy_block.svg new file mode 100644 index 0000000..3c0a65c --- /dev/null +++ b/assets/images/service_deploy_block.svg @@ -0,0 +1 @@ +UnsupportedData SourceUnsupportedETL Tool (Lineage)Entity File Creation TriggerMetadata sourcesCreate TemplateDesign Meta-ModelCreate Entities from TemplatePull Entity TemplateSynapse Parse Source PipelineSchedule TriggerCreate TypeCustom TypeCreator AppSynapse Purview Ingestion PipelineAzure PurviewADLS \ No newline at end of file diff --git a/assets/images/smss_adls_config.png b/assets/images/smss_adls_config.png new file mode 100644 index 0000000..063f85e Binary files /dev/null and b/assets/images/smss_adls_config.png differ diff --git a/assets/images/smss_config_project.png b/assets/images/smss_config_project.png new file mode 100644 index 0000000..d9b040d Binary files /dev/null and b/assets/images/smss_config_project.png differ diff --git a/assets/images/smss_oledb_config.png b/assets/images/smss_oledb_config.png new file mode 100644 index 0000000..91a1ef1 Binary files /dev/null and b/assets/images/smss_oledb_config.png differ diff --git a/assets/images/spark_logging.png b/assets/images/spark_logging.png new file mode 100644 index 0000000..9d6a90b Binary files /dev/null and b/assets/images/spark_logging.png differ diff --git a/assets/images/ssis_install.png b/assets/images/ssis_install.png new file mode 100644 index 0000000..2a41caf Binary files /dev/null and b/assets/images/ssis_install.png differ diff --git a/assets/images/ssis_pipeline.png b/assets/images/ssis_pipeline.png new file mode 100644 index 0000000..6a6dc57 Binary files /dev/null and b/assets/images/ssis_pipeline.png differ diff --git a/assets/images/synapse_edit_trigger.png b/assets/images/synapse_edit_trigger.png new file mode 100644 index 0000000..d47a3db Binary files /dev/null and b/assets/images/synapse_edit_trigger.png differ diff --git a/assets/images/synapse_select_trigger.png b/assets/images/synapse_select_trigger.png new file mode 100644 index 0000000..12833fa Binary files /dev/null and b/assets/images/synapse_select_trigger.png differ diff --git a/assets/images/synapse_start_trigger.png b/assets/images/synapse_start_trigger.png new file mode 100644 index 0000000..a0a472a Binary files /dev/null and b/assets/images/synapse_start_trigger.png differ diff --git a/assets/images/tag_db_deploy_blocks.svg b/assets/images/tag_db_deploy_blocks.svg new file mode 100644 index 0000000..59ef587 --- /dev/null +++ b/assets/images/tag_db_deploy_blocks.svg @@ -0,0 +1 @@ +Hybrid RunbookSQL VM –SSIS (Lineage)Automation AccountLog AnalyticsUnsupportedData SourceEntity File Creation TriggerCreate TemplateDesign Meta-ModelCreate Entities from TemplatePull Entity TemplateCreate TypeCustom TypeCreator AppSynapse Purview Ingestion PipelineAzure PurviewSchedule TriggerMetadata sourcesSynapse Parse Source PipelineADLS \ No newline at end of file diff --git a/assets/images/tag_db_diagram.svg b/assets/images/tag_db_diagram.svg new file mode 100644 index 0000000..8f26e0a --- /dev/null +++ b/assets/images/tag_db_diagram.svg @@ -0,0 +1 @@ +Azure PurviewCustom TypeCreator AppEntity File Creation TriggerCreate TemplateDesign Meta-ModelCreate Entities from TemplatePull Entity TemplateSynapse Parse Source PipelineSynapse Purview Ingestion PipelineSchedule TriggerCreate TypeADLSExtract Metadata (xml file)OSI PITimeSeries DB \ No newline at end of file diff --git a/assets/images/tag_db_folder_structure.svg b/assets/images/tag_db_folder_structure.svg new file mode 100644 index 0000000..7c8d9aa --- /dev/null +++ b/assets/images/tag_db_folder_structure.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/assets/images/upload_file_dialog.png b/assets/images/upload_file_dialog.png new file mode 100644 index 0000000..fc14c5b Binary files /dev/null and b/assets/images/upload_file_dialog.png differ diff --git a/assets/images/upload_file_to_cloud_console_directory.png b/assets/images/upload_file_to_cloud_console_directory.png new file mode 100644 index 0000000..ef11671 Binary files /dev/null and b/assets/images/upload_file_to_cloud_console_directory.png differ diff --git a/assets/images/upload_with_cloud_console.png b/assets/images/upload_with_cloud_console.png new file mode 100644 index 0000000..1dc48b0 Binary files /dev/null and b/assets/images/upload_with_cloud_console.png differ diff --git a/assets/images/video.gif b/assets/images/video.gif new file mode 100644 index 0000000..7254016 Binary files /dev/null and b/assets/images/video.gif differ diff --git a/assets/images/webhook_params.png b/assets/images/webhook_params.png new file mode 100644 index 0000000..aa5a767 Binary files /dev/null and b/assets/images/webhook_params.png differ diff --git a/assets/images/webhook_set_hybrid.png b/assets/images/webhook_set_hybrid.png new file mode 100644 index 0000000..5d738e7 Binary files /dev/null and b/assets/images/webhook_set_hybrid.png differ diff --git a/examples/ssis/deploy/.gitignore b/examples/ssis/deploy/.gitignore new file mode 100644 index 0000000..4501d21 --- /dev/null +++ b/examples/ssis/deploy/.gitignore @@ -0,0 +1,2 @@ +settings.sh +export_names.sh diff --git a/examples/ssis/deploy/arm/deploy_automation.json b/examples/ssis/deploy/arm/deploy_automation.json new file mode 100644 index 0000000..44df8a7 --- /dev/null +++ b/examples/ssis/deploy/arm/deploy_automation.json @@ -0,0 +1,125 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + + "parameters": { + "ps_loopscript_location": { + "type": "string" + }, + "automationName": { + "type": "string" + }, + "runbookName": { + "type": "string" + }, + "webhookName": { + "type": "string" + }, + "logAnalyticsName": { + "type": "string" + }, + "vmCredsName": { + "type": "string" + }, + "vmAdminPassword": { + "type": "SecureString" + }, + "vmAdminUserName": { + "type": "string" + } + }, + "variables": { + "location": "[resourceGroup().location]" + }, + "resources": [ + { + "type": "microsoft.operationalinsights/workspaces", + "apiVersion": "2020-10-01", + "name": "[parameters('logAnalyticsName')]", + "location": "[variables('location')]", + "properties": { + "sku": { + "name": "pergb2018" + }, + "retentionInDays": 30, + "features": { + "enableLogAccessUsingOnlyResourcePermissions": true + }, + "workspaceCapping": { + "dailyQuotaGb": -1 + }, + "publicNetworkAccessForIngestion": "Enabled", + "publicNetworkAccessForQuery": "Enabled" + } + }, + { + "type": "Microsoft.Automation/automationAccounts", + "apiVersion": "2020-01-13-preview", + "name": "[parameters('automationName')]", + "location": "[variables('location')]", + "properties": { + "sku": { + "name": "Basic" + }, + "encryption": { + "keySource": "Microsoft.Automation", + "identity": {} + } + }, + "resources":[ + { + "type": "credentials", + "apiVersion": "2020-01-13-preview", + "name": "[parameters('vmCredsName')]", + "dependsOn": [ + "[parameters('automationName')]" + ], + "properties": { + "userName": "[parameters('vmAdminUserName')]", + "password": "[parameters('vmAdminPassword')]" + } + }, + { + "type": "runbooks", + "apiVersion": "2019-06-01", + "name": "[parameters('runbookName')]", + "location": "[variables('location')]", + "dependsOn": [ + "[parameters('automationName')]" + ], + "properties": { + "runbookType": "PowerShell", + "logVerbose": false, + "logProgress": false, + "publishContentLink": { + "uri":"[parameters('ps_loopscript_location')]", + "version": "1.0.0.0" + } + } + }, + { + "type": "webhooks", + "apiVersion": "2015-10-31", + "name": "[parameters('webhookName')]", + "dependsOn": [ + "[parameters('automationName')]", + "[parameters('runbookName')]" + ], + "properties": { + "isEnabled": true, + "expiryTime": "2022-05-17T17:43:23.955+00:00", + "runbook": { + "name": "[parameters('runbookName')]" + } + } + } + ] + } + ], + "outputs": { + "webhookUri": { + "type": "String", + "value": "[reference(parameters('webhookName')).uri]" + } + } +} \ No newline at end of file diff --git a/examples/ssis/deploy/arm/deploy_sql_server_vm.json b/examples/ssis/deploy/arm/deploy_sql_server_vm.json new file mode 100644 index 0000000..e4fbe82 --- /dev/null +++ b/examples/ssis/deploy/arm/deploy_sql_server_vm.json @@ -0,0 +1,404 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "workSpaceId": { + "type": "String", + "metadata": { + "description": "Id of the Log Analytics workspace for the VM extension" + } + }, + "workSpaceIdKey": { + "type": "String", + "metadata": { + "description": "Key of the Log Analytics workspace for the VM extension" + } + }, + "existingVirtualNetworkName": { + "type": "String", + "metadata": { + "description": "Specify the name of an existing VNet in the same resource group" + } + }, + "existingSubnetName": { + "type": "String", + "metadata": { + "description": "Specify the name of the Subnet Name" + } + }, + "adminUsername": { + "type": "String", + "metadata": { + "description": "The admin user name of the VM" + } + }, + "adminPassword": { + "type": "SecureString", + "metadata": { + "description": "The admin password of the VM" + } + }, + "existingVnetResourceGroup": { + "type": "String", + "defaultValue": "[resourceGroup().name]", + "metadata": { + "description": "Specify the resource group of the existing VNet" + } + }, + "virtualMachineName": { + "type": "String", + "metadata": { + "description": "The name of the VM" + } + }, + "virtualMachineSize": { + "type": "String", + "defaultValue": "Standard_D8s_v3", + "metadata": { + "description": "The virtual machine size." + } + }, + "imageOffer": { + "type": "String", + "defaultValue": "sql2019-ws2019", + "allowedValues": [ + "sql2019-ws2019", + "sql2017-ws2019", + "SQL2017-WS2016", + "SQL2016SP1-WS2016", + "SQL2016SP2-WS2016", + "SQL2014SP3-WS2012R2", + "SQL2014SP2-WS2012R2" + ], + "metadata": { + "description": "Windows Server and SQL Offer" + } + }, + "sqlSku": { + "type": "String", + "defaultValue": "Standard", + "allowedValues": [ + "Standard", + "Enterprise", + "SQLDEV", + "Web", + "Express" + ], + "metadata": { + "description": "SQL Server Sku" + } + }, + "storageWorkloadType": { + "type": "String", + "defaultValue": "General", + "allowedValues": [ + "General", + "OLTP", + "DW" + ], + "metadata": { + "description": "SQL Server Workload Type" + } + }, + "sqlDataDisksCount": { + "type": "int", + "defaultValue": 1, + "minValue": 1, + "maxValue": 8, + "metadata": { + "description": "Amount of data disks (1TB each) for SQL Data files" + } + }, + "dataPath": { + "type": "String", + "defaultValue": "F:\\SQLData", + "metadata": { + "description": "Path for SQL Data files. Please choose drive letter from F to Z, and other drives from A to E are reserved for system" + } + }, + "sqlLogDisksCount": { + "type": "int", + "defaultValue": 1, + "minValue": 1, + "maxValue": 8, + "metadata": { + "description": "Amount of data disks (1TB each) for SQL Log files" + } + }, + "logPath": { + "type": "String", + "defaultValue": "G:\\SQLLog", + "metadata": { + "description": "Path for SQL Log files. Please choose drive letter from F to Z and different than the one used for SQL data. Drive letter from A to E are reserved for system" + } + }, + "location": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "Location for all resources." + } + } + }, + "variables": { + "networkInterfaceName": "[concat(parameters('virtualMachineName'), '-nic')]", + "networkSecurityGroupName": "[concat(parameters('virtualMachineName'), '-nsg')]", + "networkSecurityGroupRules": [ + { + "name": "RDP", + "properties": { + "priority": 300, + "protocol": "TCP", + "access": "Allow", + "direction": "Inbound", + "sourceAddressPrefix": "*", + "sourcePortRange": "*", + "destinationAddressPrefix": "*", + "destinationPortRange": "3389" + } + }, + { + "name": "Sql-Port-1433", + "properties": { + "protocol": "*", + "sourcePortRange": "*", + "destinationPortRange": "1433", + "sourceAddressPrefix": "AzureCloud", + "destinationAddressPrefix": "*", + "access": "Allow", + "priority": 310, + "direction": "Inbound", + "sourcePortRanges": [], + "destinationPortRanges": [], + "sourceAddressPrefixes": [], + "destinationAddressPrefixes": [] + } + }, + { + "name": "SSIS-port-3342", + "properties": { + "protocol": "*", + "sourcePortRange": "*", + "destinationPortRange": "3342", + "sourceAddressPrefix": "AzureCloud", + "destinationAddressPrefix": "*", + "access": "Allow", + "priority": 320, + "direction": "Inbound", + "sourcePortRanges": [], + "destinationPortRanges": [], + "sourceAddressPrefixes": [], + "destinationAddressPrefixes": [] + } + } + ], + "publicIpAddressName": "[concat(parameters('virtualMachineName'), '-publicip-', uniqueString(parameters('virtualMachineName')))]", + "publicIpAddressType": "Dynamic", + "publicIpAddressSku": "Basic", + "diskConfigurationType": "NEW", + "nsgId": "[resourceId('Microsoft.Network/networkSecurityGroups', variables('networkSecurityGroupName'))]", + "subnetRef": "[resourceID(parameters('existingVNetResourceGroup'), 'Microsoft.Network/virtualNetWorks/subnets', parameters('existingVirtualNetworkName'), parameters('existingSubNetName'))]", + "dataDisksLuns": "[array(range(0 ,parameters('sqlDataDisksCount')))]", + "logDisksLuns": "[array(range(parameters('sqlDataDisksCount'), parameters('sqlLogDisksCount')))]", + "dataDisks": { + "createOption": "empty", + "caching": "ReadOnly", + "writeAcceleratorEnabled": false, + "storageAccountType": "Premium_LRS", + "diskSizeGB": 1023 + }, + "tempDbPath": "D:\\SQLTemp" + }, + "resources": [ + { + "type": "Microsoft.Network/publicIpAddresses", + "apiVersion": "2020-06-01", + "name": "[variables('publicIpAddressName')]", + "location": "[parameters('location')]", + "sku": { + "name": "[variables('publicIpAddressSku')]" + }, + "properties": { + "publicIpAllocationMethod": "[variables('publicIpAddressType')]" + } + }, + { + "type": "Microsoft.Network/networkSecurityGroups", + "apiVersion": "2020-06-01", + "name": "[variables('networkSecurityGroupName')]", + "location": "[parameters('location')]", + "properties": { + "securityRules": "[variables('networkSecurityGroupRules')]" + } + }, + { + "type": "Microsoft.Network/networkInterfaces", + "apiVersion": "2020-06-01", + "name": "[variables('networkInterfaceName')]", + "location": "[parameters('location')]", + "dependsOn": [ + "[resourceId('Microsoft.Network/networkSecurityGroups/', variables('networkSecurityGroupName'))]", + "[resourceId('Microsoft.Network/publicIpAddresses/', variables('publicIpAddressName'))]" + ], + "properties": { + "ipConfigurations": [ + { + "name": "ipconfig1", + "properties": { + "subnet": { + "id": "[variables('subnetRef')]" + }, + "privateIPAllocationMethod": "Dynamic", + "publicIpAddress": { + "id": "[resourceId('Microsoft.Network/publicIpAddresses', variables('publicIpAddressName'))]" + } + } + } + ], + "enableAcceleratedNetworking": true, + "networkSecurityGroup": { + "id": "[variables('nsgId')]" + } + } + }, + { + "type": "Microsoft.Compute/virtualMachines", + "apiVersion": "2020-06-01", + "name": "[parameters('virtualMachineName')]", + "location": "[parameters('location')]", + "dependsOn": [ + "[resourceId('Microsoft.Network/networkInterfaces/', variables('networkInterfaceName'))]" + ], + "properties": { + "hardwareProfile": { + "vmSize": "[parameters('virtualMachineSize')]" + }, + "storageProfile": { + "osDisk": { + "createOption": "fromImage", + "managedDisk": { + "storageAccountType": "Premium_LRS" + } + }, + "imageReference": { + "publisher": "MicrosoftSQLServer", + "offer": "[parameters('imageOffer')]", + "sku": "[parameters('sqlSku')]", + "version": "latest" + }, + "copy": [ + { + "name": "dataDisks", + "count": "[add(parameters('sqlDataDisksCount'), parameters('sqlLogDisksCount'))]", + "input": { + "lun": "[copyIndex('dataDisks')]", + "createOption": "[variables('dataDisks').createOption]", + "caching": "[if(greaterOrEquals(copyIndex('dataDisks'), parameters('sqlDataDisksCount')) ,'None', variables('dataDisks').caching )]", + "writeAcceleratorEnabled": "[variables('dataDisks').writeAcceleratorEnabled]", + "diskSizeGB": "[variables('dataDisks').diskSizeGB]", + "managedDisk": { + "storageAccountType": "[variables('dataDisks').storageAccountType]" + } + } + } + ] + }, + "networkProfile": { + "networkInterfaces": [ + { + "id": "[resourceId('Microsoft.Network/networkInterfaces', variables('networkInterfaceName'))]" + } + ] + }, + "osProfile": { + "computerName": "[parameters('virtualMachineName')]", + "adminUsername": "[parameters('adminUsername')]", + "adminPassword": "[parameters('adminPassword')]", + "windowsConfiguration": { + "enableAutomaticUpdates": true, + "provisionVmAgent": true + } + } + }, + "resources": [ + { + "type": "extensions", + "name": "OMSExtension", + "apiVersion": "2020-12-01", + "location": "[resourceGroup().location]", + "dependsOn": [ + "[resourceId('Microsoft.Compute/virtualMachines', parameters('virtualMachineName'))]" + ], + "properties": { + "publisher": "Microsoft.EnterpriseCloud.Monitoring", + "type": "MicrosoftMonitoringAgent", + "typeHandlerVersion": "1.0", + "autoUpgradeMinorVersion": true, + "settings": { + "workspaceId": "[parameters('workSpaceId')]" + }, + "protectedSettings": { + "workspaceKey": "[parameters('workSpaceIdKey')]" + } + } + } + ] + }, + { + "type": "Microsoft.SqlVirtualMachine/SqlVirtualMachines", + "apiVersion": "2017-03-01-preview", + "name": "[parameters('virtualMachineName')]", + "location": "[parameters('location')]", + "dependsOn": [ + "[resourceId('Microsoft.Compute/virtualMachines', parameters('virtualMachineName'))]" + ], + "properties": { + "virtualMachineResourceId": "[resourceId('Microsoft.Compute/virtualMachines', parameters('virtualMachineName'))]", + "sqlManagement": "Full", + "SqlServerLicenseType": "PAYG", + "serverConfigurationsManagementSettings": { + "sqlConnectivityUpdateSettings": { + "connectivityType": "PRIVATE", + "port": "1433", + "sqlAuthUpdateUserName": "[parameters('adminUsername')]", + "sqlAuthUpdatePassword": "[parameters('adminPassword')]" + } + }, + "StorageConfigurationSettings": { + "DiskConfigurationType": "[variables('diskConfigurationType')]", + "StorageWorkloadType": "[parameters('storageWorkloadType')]", + "SQLDataSettings": { + "LUNs": "[variables('dataDisksLUNs')]", + "DefaultFilePath": "[parameters('dataPath')]" + }, + "SQLLogSettings": { + "Luns": "[variables('logDisksLUNs')]", + "DefaultFilePath": "[parameters('logPath')]" + }, + "SQLTempDbSettings": { + "DefaultFilePath": "[variables('tempDbPath')]" + } + } + } + }, + { + "apiVersion": "2020-10-01", + "name": "pid-436d2bea-3759-4494-b63b-aa95d0407e1f", + "type": "Microsoft.Resources/deployments", + "properties": { + "mode": "Incremental", + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "resources": [] + } + } + } + ], + "outputs": { + "adminUsername": { + "type": "String", + "value": "[parameters('adminUsername')]" + } + } + } \ No newline at end of file diff --git a/examples/ssis/deploy/arm/deploy_vm_networking.json b/examples/ssis/deploy/arm/deploy_vm_networking.json new file mode 100644 index 0000000..355f5d9 --- /dev/null +++ b/examples/ssis/deploy/arm/deploy_vm_networking.json @@ -0,0 +1,136 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + + "parameters": { + "publicIpAddressName": { + "type": "String" + }, + "networkSecurityGroupName": { + "type": "String" + }, + "networkInterfaceName": { + "type": "String" + }, + "vnetName": { + "type": "String" + }, + "subnetName": { + "type": "String" + } + }, + + "variables": { + "location": "[resourceGroup().location]" + }, + + "resources": [ + { + "type": "Microsoft.Network/publicIpAddresses", + "apiVersion": "2020-06-01", + "name": "[parameters('publicIpAddressName')]", + "location": "[variables('location')]", + "sku": { + "name": "Basic", + "tier": "Regional" + }, + "properties": { + "publicIpAllocationMethod": "Static" + } + }, + { + "type": "Microsoft.Network/virtualNetworks", + "apiVersion": "2020-11-01", + "name": "[parameters('vnetName')]", + "location": "[variables('location')]", + "properties": { + "addressSpace": { + "addressPrefixes": [ + "172.17.0.0/16" + ] + }, + "subnets": [ + { + "name": "default", + "properties": { + "addressPrefix": "172.17.0.0/24", + "delegations": [], + "privateEndpointNetworkPolicies": "Enabled", + "privateLinkServiceNetworkPolicies": "Enabled" + } + }, + { + "name": "[parameters('subnetName')]", + "properties": { + "addressPrefix": "172.17.1.0/24", + "serviceEndpoints": [ + { + "service": "Microsoft.Sql", + "locations": [ + "[variables('location')]" + ] + } + ], + "delegations": [], + "privateEndpointNetworkPolicies": "Enabled", + "privateLinkServiceNetworkPolicies": "Enabled" + } + } + ], + "virtualNetworkPeerings": [], + "enableDdosProtection": false + } + }, + { + "type": "Microsoft.Network/virtualNetworks/subnets", + "apiVersion": "2020-11-01", + "name": "[concat(parameters('vnetName'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Network/virtualNetworks', parameters('vnetName'))]" + ], + "properties": { + "addressPrefix": "172.17.0.0/24", + "delegations": [], + "privateEndpointNetworkPolicies": "Enabled", + "privateLinkServiceNetworkPolicies": "Enabled" + } + }, + { + "type": "Microsoft.Network/virtualNetworks/subnets", + "apiVersion": "2020-11-01", + "name": "[concat(parameters('vnetName'), '/', parameters('subnetName'))]", + "dependsOn": [ + "[resourceId('Microsoft.Network/virtualNetworks', parameters('vnetName'))]" + ], + "properties": { + "addressPrefix": "172.17.1.0/24", + "serviceEndpoints": [ + { + "service": "Microsoft.Sql", + "locations": [ + "westus2" + ] + } + ], + "delegations": [], + "privateEndpointNetworkPolicies": "Enabled", + "privateLinkServiceNetworkPolicies": "Enabled" + } + }, + { + "apiVersion": "2020-10-01", + "name": "pid-436d2bea-3759-4494-b63b-aa95d0407e1f", + "type": "Microsoft.Resources/deployments", + "properties": { + "mode": "Incremental", + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "resources": [] + } + } + } + ] +} + + \ No newline at end of file diff --git a/examples/ssis/deploy/deploy_ssis.md b/examples/ssis/deploy/deploy_ssis.md new file mode 100644 index 0000000..bf6ac86 --- /dev/null +++ b/examples/ssis/deploy/deploy_ssis.md @@ -0,0 +1,174 @@ +# Deploy ETL Lineage Example (SSIS) + +## Components + +![PCCSA Block Diagram](../../../assets/images/pccsa_ssis_block_diagram.svg) + +## Prerequisite + +Follow the instructions for deploying the base solution under [purview_connector_services](../../../purview_connector_services/deploy/deploy_sa.md) + +## Run Installation Script + +* Fill in the secrets file with virtual machine credentials. Use 'vmadmin' for the user name. +* To run script + * Start the cloud CLI in bash mode + * cd to the cloud storage directory (clouddrive) + * upload the settings.sh file (created above) to the examples/ssis/deploy directory using the Upload/Download files tool + + ![upload files](../../../assets/images/upload_with_cloud_console.png) + + * Choose the "Manage file share" option, navigate to the PurviewACC/examples/ssis/deploy directory and copy the settings.sh file into this directory + + ![cloud console directory](../../../assets/images/upload_file_to_cloud_console_directory.png) + + ![upload file dialog](../../../assets/images/upload_file_dialog.png) + + * navigate to the PurviewACC/examples/ssis/deploy directory and run the deploy_ssis.sh script (./deploy_ssis.sh) + +## Configure the SQL Server VM + +* RDP into the VM using the username and password from your settings.sh file +* If asked to publish identity to the network, select 'Yes' +* From the Server -> Local Server pane, turn off IE Enhanced Security Mode Configuration +* [Install .NET Framework 4.8 or later](https://dotnet.microsoft.com/download/dotnet-framework/net48) Runtime (requires reboot of the VM) +* Install the necessary PowerShell modules for running the Hybrid Runbook - if prompted, select 'Yes' in all cases + * Start PowerShell as Admin + * Make sure you have the latest version of PowerShellGet + + ```PowerShell + Install-Module -Name PowerShellGet -Force + ``` + + * Set the appropriate execution policy + + ```PowerShell + Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser + ``` + + * Install the Az CLI - this takes some time, you can go to the next section and install SSIS Services in parallel + + _Note: This installation can appear to hang. If it runs for a significant time after the progress indicators are shown, press any key in the command window to continue._ + + ```PowerShell + Install-Module -Name Az -Scope CurrentUser -Repository PSGallery -Force + ``` + + * Install the Az storage module + + ```PowerShell + Install-Module -Name Az.Storage -Force + ``` + + * Install the SQL Server module + + ```PowerShell + Install-Module -Name SqlServer -AllowClobber + ``` + +* Install SSIS Services + * Open SQL Server Management Studio by typing the name at the start menu + * Connect to the local server using Windows authentication (should be the default option) + * Right click the integration services catalogs item and choose "Create catalog" + * Enable CLR integration and running SSIS sp at start up, type a password and choose "OK" + + ![SSIS Instal Diagram](../../../assets/images/ssis_install.png) + + * Install the [SSIS Azure Feature Pack](https://www.microsoft.com/en-us/download/details.aspx?id=100430) (x64 version). Be sure you have disabled IE enhanced security from the Server panel as mentioned above +* Sample Data + * Copy all .sql files under PurviewACC\examples\ssis\example_data to a folder on the virutal machine (e.g. c:\sql_files) + * On the VM, within SSMS, choose File->Open->File and open the CreateDb.sql script. Execute the script to create the purview-sqlmi-db database + * Repeat the above step running the CreateCustomer.sql and CreateMovies.sql scripts +* Import SSIS Project + * Open file explorer and create an ssis_packages directory under c: + * Copy the PurviewACC/examples/ssis/example_ssis_project/simple_example_ssis_project.ispac and paste into the ssis_packages directory you created on the VM + * Use [isdeploymentwizard.exe](https://docs.microsoft.com/en-us/sql/integration-services/packages/deploy-integration-services-ssis-projects-and-packages?view=sql-server-ver15) (Find by typing "isdeploymentwizard at start menu) to import the project + * Go to "Select Source", select "Project Deployment" and browse for the .ispac file you created above. Select "Next" + * Under Select Deployment Target, choose SSIS in SQL Server to mimic on prem SQL SSIS environment + * Under Select Destination choose to browse for the server name and select the local instance + * Choose to browse for the path, select SSISDB and then choose to create a new folder. Call it 'ssistest' and choose "OK" (The name is important as it must match the name in the Synapse pipeline FolderName parameter) + * Choose next to review your options, then choose "Deploy" + +## Automation Services + +* On the VM + * Most of the automation configuration has been completed for you by scripts. The remaining step must be done while remoted into the VM + * Open a PowerShell session in Administrator mode and run the following commands to import the module. Replace version in the path with the version number appearing in your filesystem + + ```bash + cd "C:\Program Files\Microsoft Monitoring Agent\Agent\AzureAutomation\\HybridRegistration" + Import-Module .\HybridRegistration.psd1 + + ``` + + * Run the Add-HybridRunbookWorker cmdlet specifying the values for the parameters Url, Key, and GroupName (hybrid group) from the Automation Account -> Keys pane in the Azure Portal. You can choose your own value for GroupName + + ```bash + Add-HybridRunbookWorker –GroupName -Url -Key + ``` + +The Hybrid Runbook works like SHIR does for native connectors. It runs the Powershell script using the Azure Log Analytics agent on the VM or on premises server. + +* In the Azure Portal + * Change the Webhook to run a Hybrid runbook in the newly created hybrid group + * In Azure Portal open the automation account + * Go to 'Runbooks' and open your specific runbook + * Go to Webhooks link in the left pane + +![select_web_hook.png](../../../assets/images/select_web_hook.png) + +* + * + * Select the Webhook + * Open Parameters and run settings + +![webhook_params.png](../../../assets/images/webhook_params.png) + +* + * + * Select "Hybrid Worker" under "Run Settings" + * Choose the Hybrid Worker group from the drop down + +![webhook_set_hybrid.png](../../../assets/images/webhook_set_hybrid.png) + +* + * + * Be sure to hit the 'Save' button in the next screen + * Click into the Hybrid Worker Group -> Hybrid worker group settings and set it to use the existing custom credentials + +![webhook_set_hybrid.png](../../../assets/images/runbook_credentials.png) + +### Hybrid Runbook references + +* [Configure identity](https://blog.ctglobalservices.com/azure/jgs/azure-automation-setting-run-as-account-on-hybrid-worker-groups/) +* [General guidance](https://docs.microsoft.com/en-us/azure/automation/automation-windows-hrw-install) +* [Manual Configuration Step by Step](https://docs.microsoft.com/en-us/azure/automation/automation-windows-hrw-install#manual-deployment) + +## Reference - script actions + +* Pull in service names and data from base install +* Pull in VM credentials +* Add SSIS Example directories to storage +* Upload PowerShell scripts to storage - for use in Automation Runbooks +* Deploy Log Analytics, Azure Automation, Runbook, and Webhook +* Enable Log Analytics account for hybrid automation +* Store VM credentials in KeyVault +* Deploy VM networking + * Public IP Address + * Network security group + * Network interface + * VNet + * SubNet +* Deploy VM + * Standard DS13 V2 + * SQL Server 2019 + SSIS, SSMS, etc. + * Automation Hybrid Runbook agent extension + * Save VM name, and webhook uri +* Synapse + * Datasets + * Notebooks + * Pipeline + +## Privacy + +To opt out of information collection as described in [privacy.md](../../../PRIVACY.md), remove the described section from all templates in the PurviewACC/examples/ssis/deploy/arm directory. diff --git a/examples/ssis/deploy/deploy_ssis.sh b/examples/ssis/deploy/deploy_ssis.sh new file mode 100644 index 0000000..63337f0 --- /dev/null +++ b/examples/ssis/deploy/deploy_ssis.sh @@ -0,0 +1,114 @@ +#!/bin/bash + +# dynamically load missing az dependancies without prompting +az config set extension.use_dynamic_install=yes_without_prompt --output none + +# This script requires contributor and user access administrator permissions to run +source ../../../purview_connector_services/deploy/export_names.sh +source ./settings.sh + +# To run in Azure Cloud CLI, comment this section out +# az login --output none +# az account set --subscription "Early Access Engineering Subscription" --output none + +# Upload storage folders and data +# Create storage dir structure +echo "Create storage folders" +mkdir ssis-connector; cd ssis-connector; mkdir execution-id; touch ./execution-id/tmp; mkdir ssis-package;\ +touch ./ssis-package/tmp; mkdir ssis-type-templates; touch ./ssis-type-templates/tmp; mkdir working;\ +touch ./working/tmp; mkdir example-data; touch ./example-data/tmp; cd .. +# Upload dir structure to storage +az storage fs directory upload -f pccsa --account-name $storage_name -s ./ssis-connector -d ./pccsa_main --recursive --output none >> log_ssis_deploy.txt +# Remove tmp directory structure +rm -r ssis-connector +# Create public script container +az storage container create --account-name $storage_name -n scripts --public-access blob >> log_ssis_deploy.txt +az storage fs file upload -f scripts --account-name $storage_name -s ../scripts/powershell_loop_script.ps1 -p ./powershell_loop_script.ps1 --auth-mode login >> log_ssis_deploy.txt + +# upload example data +az storage fs file upload -f pccsa --account-name $storage_name -s ../example_data/MovCustomers.csv -p ./pccsa_main/ssis-connector/example-data/MovCustomers.csv --auth-mode login >> log_ssis_deploy.txt + +# upload templates +az storage fs file upload -f pccsa --account-name $storage_name -s ../meta_model/example_templates/legacy_ssis_package_process_template.json -p ./pccsa_main/ssis-connector/ssis-type-templates/legacy_ssis_package_process_template.json --auth-mode login +az storage fs file upload -f pccsa --account-name $storage_name -s ../meta_model/example_templates/legacy_ssis_package_template.json -p ./pccsa_main/ssis-connector/ssis-type-templates/legacy_ssis_package_template.json --auth-mode login + +# deploy automation +echo "deploy log analytics and automation" +# upload secrets to keyvault +az keyvault secret set --vault-name $key_vault_name --name vm-password --value $vm_password --output none >> log_ssis_deploy.txt +# setup service names +ps_loopscript_location="https://$storage_name.blob.core.windows.net/scripts/powershell_loop_script.ps1" +automationName=$prefix"automation"$suffix +logAnalyticsName=$prefix"loganalytics"$suffix +runbookName=$prefix"runbook"$suffix +webhookName=$prefix"webhook"$suffix +vmCredsName=$prefix"vmcreds"$suffix +# get keyvault reference +refid_keyvault="/subscriptions/$subscription_id/resourceGroups/$resource_group/providers/Microsoft.KeyVault/vaults/$key_vault_name" +# setup params +params="{\"ps_loopscript_location\":{\"value\":\"$ps_loopscript_location\"},"\ +"\"automationName\":{\"value\":\"$automationName\"},"\ +"\"logAnalyticsName\":{\"value\":\"$logAnalyticsName\"},"\ +"\"vmAdminUserName\":{\"value\":\"$vmAdminUserName\"},"\ +"\"vmCredsName\":{\"value\":\"$vmCredsName\"},"\ +"\"runbookName\":{\"value\":\"$runbookName\"},"\ +"\"webhookName\":{\"value\":\"$webhookName\"},"\ +"\"vmAdminPassword\":{\"reference\":{\"keyVault\":{\"id\":\"$refid_keyvault\"},\"secretName\":\"vm-password\"}}}" +# deploy - hybrid runbook worker group must be manually configured: https://docs.microsoft.com/en-us/azure/automation/automation-windows-hrw-install#manual-deployment +web_hook_uri=$(az deployment group create --resource-group $resource_group --parameters $params --template-file ./arm/deploy_automation.json --query properties.outputs.webhookUri.value) + +# deploy vm networking +echo "deploy vm networking" +# setup service names +publicIpAddressName=$prefix"pubip"$suffix +networkSecurityGroupName=$prefix"nsg"$suffix +networkInterfaceName=$prefix"nic"$suffix +vnetName=$prefix"vnet"$suffix +subnetName=$prefix"subnet"$suffix +# setup params +params="{\"publicIpAddressName\":{\"value\":\"$publicIpAddressName\"},"\ +"\"networkSecurityGroupName\":{\"value\":\"$networkSecurityGroupName\"},"\ +"\"networkInterfaceName\":{\"value\":\"$networkInterfaceName\"},"\ +"\"vnetName\":{\"value\":\"$vnetName\"},"\ +"\"subnetName\":{\"value\":\"$subnetName\"}}" +# deploy +az deployment group create --resource-group $resource_group --parameters $params --template-file ./arm/deploy_vm_networking.json --output none >> log_ssis_deploy.txt + +# deploy sql server vm +echo "deploy sql server vm" +# setup service names +vm_name=$prefix"vm"$suffix +# vm name cannot be more than 15 chars long +vm_name=${vm_name:0:15} + +# Get workspace info for vm extension +workspace_id=$(az monitor log-analytics workspace show --resource-group $resource_group --workspace-name $logAnalyticsName --query customerId -o tsv) +workspace_key=$(az monitor log-analytics workspace get-shared-keys --resource-group $resource_group --workspace-name $logAnalyticsName --query primarySharedKey -o tsv) +# setup service names +existingSubnetName=$vnetName/$subnetName +# setup params +params="{\"workSpaceId\":{\"value\":\"$workspace_id\"},"\ +"\"workSpaceIdKey\":{\"value\":\"$workspace_key\"},"\ +"\"virtualMachineName\":{\"value\":\"$vm_name\"},"\ +"\"existingVirtualNetworkName\":{\"value\":\"$vnetName\"},"\ +"\"existingSubnetName\":{\"value\":\"$subnetName\"},"\ +"\"adminUsername\":{\"value\":\"$vmAdminUserName\"},"\ +"\"adminPassword\":{\"reference\":{\"keyVault\":{\"id\":\"$refid_keyvault\"},\"secretName\":\"vm-password\"}}}" +# deploy +az deployment group create --resource-group $resource_group --parameters $params --template-file ./arm/deploy_sql_server_vm.json --output none >> log_ssis_deploy.txt + +# Azure VM info +vm_public_ip=$(az vm show -d -g $resource_group -n $vm_name --query publicIps -o tsv) +sql_con_str="data source = localhost; initial catalog = SSISDB; trusted_connection = true; user id = $vmAdminUserName; password = $vm_password" + +az keyvault secret set --vault-name $key_vault_name --name "sql-connection-string" --value "$sql_con_str" --output none >> log_ssis_deploy.txt + +echo vm_name=$vm_name >> ../../../purview_connector_services/deploy/export_names.sh +echo web_hook_uri=$web_hook_uri >> ../../../purview_connector_services/deploy/export_names.sh +echo vm_public_ip=$vm_public_ip >> ../../../purview_connector_services/deploy/export_names.sh + +# Enable log analytics agent for hybrid automation runbook +az monitor log-analytics workspace pack enable --name AzureAutomation --resource-group $resource_group --workspace-name $logAnalyticsName >> log_ssis_deploy.txt + +./deploy_ssis_synapse.sh + diff --git a/examples/ssis/deploy/deploy_ssis_synapse.sh b/examples/ssis/deploy/deploy_ssis_synapse.sh new file mode 100644 index 0000000..db68282 --- /dev/null +++ b/examples/ssis/deploy/deploy_ssis_synapse.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# Pull in base deploy service names +source ../../../purview_connector_services/deploy/export_names.sh +source ./settings.sh + +# Configure Synapse Notebooks and Pipelines + +# Data set +az synapse dataset create --workspace-name $synapse_name --name Xml1 --file @../synapse/dataset/Xml1.json >> log_ssis_deploy.txt +az synapse dataset create --workspace-name $synapse_name --name Xml2 --file @../synapse/dataset/Xml2.json >> log_ssis_deploy.txt +az synapse dataset create --workspace-name $synapse_name --name Json1 --file @../synapse/dataset/Json1.json >> log_ssis_deploy.txt + +# Notebook +az synapse notebook create --workspace-name $synapse_name --spark-pool-name notebookrun --name SSIS_Scan_Package --file @../synapse/notebook/SSIS_Scan_Package.ipynb >> log_ssis_deploy.txt +az synapse notebook create --workspace-name $synapse_name --spark-pool-name notebookrun --name SSISDB_Get_Params --file @../synapse/notebook/SSISDB_Get_Params.ipynb >> log_ssis_deploy.txt + +# Gather pipeline data +client_secret_uri=$(az keyvault secret show --name client-secret --vault-name $key_vault_name --query 'id' -o tsv) +sql_con_str_uri=$(az keyvault secret show --name sql-connection-string --vault-name $key_vault_name --query 'id' -o tsv) +storage_account_key_uri=$(az keyvault secret show --name storage-account-key --vault-name $key_vault_name --query 'id' -o tsv) +tag_sql_pwd_uri=$(az keyvault secret show --name vm-password --vault-name $key_vault_name --query 'id' -o tsv) + +# Pipeline +echo "Creating SSIS pipeline" +# Build multiple substitutions for pipeline json - note sed can use any delimeter so url changed to avoid conflict with slash char +pipeline_sub="s//$vm_public_ip/;"\ +"s//$storage_name/;"\ +"s//$key_vault_name/;"\ +"s//$vmAdminUserName/;"\ +"s@@$web_hook_uri@;"\ +"s@@$sql_con_str_uri@;"\ +"s@@$storage_account_key_uri@;"\ +"s@@$tag_sql_pwd_uri@" +sed $pipeline_sub '../synapse/pipeline/SSIS_Package_Pipeline.json' > '../synapse/pipeline/SSIS_Package_Pipeline-tmp.json' +# Create pipeline in Synapse +az synapse pipeline create --workspace-name $synapse_name --name 'SSIS_Package_Pipeline' --file '@../synapse/pipeline/SSIS_Package_Pipeline-tmp.json' --output none >> log_ssis_deploy.txt +# Delete the tmp json +rm '../synapse/pipeline/SSIS_Package_Pipeline-tmp.json' \ No newline at end of file diff --git a/examples/ssis/deploy/settings.sh.rename b/examples/ssis/deploy/settings.sh.rename new file mode 100644 index 0000000..361af4c --- /dev/null +++ b/examples/ssis/deploy/settings.sh.rename @@ -0,0 +1,4 @@ +#!/bin/bash + +vmAdminUserName="" +vm_password="" \ No newline at end of file diff --git a/examples/ssis/example_data/CreateCustomers.sql b/examples/ssis/example_data/CreateCustomers.sql new file mode 100644 index 0000000..4456791 --- /dev/null +++ b/examples/ssis/example_data/CreateCustomers.sql @@ -0,0 +1,24 @@ +USE [purview-sqlmi-db] +GO + +/****** Object: Table [dbo].[MovCustomers] Script Date: 7/27/2021 3:56:43 AM ******/ +SET ANSI_NULLS ON +GO + +SET QUOTED_IDENTIFIER ON +GO + +CREATE TABLE [dbo].[MovCustomers]( + [Customer_ID] [nchar](255) NULL, + [Last_Name] [nchar](255) NULL, + [First_Name] [nchar](255) NULL, + [Addr_1] [nchar](255) NULL, + [Addr_2] [nchar](255) NULL, + [City] [nchar](255) NULL, + [State] [nchar](255) NULL, + [Zip_Code] [nchar](255) NULL, + [Phone_Number] [nchar](255) NULL, + [Created_Date] [smalldatetime] NULL, + [Updated_Date] [smalldatetime] NULL +) ON [PRIMARY] +GO \ No newline at end of file diff --git a/examples/ssis/example_data/CreateDb.sql b/examples/ssis/example_data/CreateDb.sql new file mode 100644 index 0000000..323d16e --- /dev/null +++ b/examples/ssis/example_data/CreateDb.sql @@ -0,0 +1,114 @@ +USE [master] +GO + +/****** Object: Database [purview-sqlmi-db] Script Date: 7/27/2021 10:57:37 AM ******/ +CREATE DATABASE [purview-sqlmi-db] + CONTAINMENT = NONE + ON PRIMARY +( NAME = N'purview-sqlmi-db', FILENAME = N'F:\SQLData\purview-sqlmi-db.mdf' , SIZE = 8192KB , MAXSIZE = UNLIMITED, FILEGROWTH = 65536KB ) + LOG ON +( NAME = N'purview-sqlmi-db_log', FILENAME = N'G:\SQLLog\purview-sqlmi-db_log.ldf' , SIZE = 8192KB , MAXSIZE = 2048GB , FILEGROWTH = 65536KB ) + WITH CATALOG_COLLATION = DATABASE_DEFAULT +GO + +IF (1 = FULLTEXTSERVICEPROPERTY('IsFullTextInstalled')) +begin +EXEC [purview-sqlmi-db].[dbo].[sp_fulltext_database] @action = 'enable' +end +GO + +ALTER DATABASE [purview-sqlmi-db] SET ANSI_NULL_DEFAULT OFF +GO + +ALTER DATABASE [purview-sqlmi-db] SET ANSI_NULLS OFF +GO + +ALTER DATABASE [purview-sqlmi-db] SET ANSI_PADDING OFF +GO + +ALTER DATABASE [purview-sqlmi-db] SET ANSI_WARNINGS OFF +GO + +ALTER DATABASE [purview-sqlmi-db] SET ARITHABORT OFF +GO + +ALTER DATABASE [purview-sqlmi-db] SET AUTO_CLOSE OFF +GO + +ALTER DATABASE [purview-sqlmi-db] SET AUTO_SHRINK OFF +GO + +ALTER DATABASE [purview-sqlmi-db] SET AUTO_UPDATE_STATISTICS ON +GO + +ALTER DATABASE [purview-sqlmi-db] SET CURSOR_CLOSE_ON_COMMIT OFF +GO + +ALTER DATABASE [purview-sqlmi-db] SET CURSOR_DEFAULT GLOBAL +GO + +ALTER DATABASE [purview-sqlmi-db] SET CONCAT_NULL_YIELDS_NULL OFF +GO + +ALTER DATABASE [purview-sqlmi-db] SET NUMERIC_ROUNDABORT OFF +GO + +ALTER DATABASE [purview-sqlmi-db] SET QUOTED_IDENTIFIER OFF +GO + +ALTER DATABASE [purview-sqlmi-db] SET RECURSIVE_TRIGGERS OFF +GO + +ALTER DATABASE [purview-sqlmi-db] SET DISABLE_BROKER +GO + +ALTER DATABASE [purview-sqlmi-db] SET AUTO_UPDATE_STATISTICS_ASYNC OFF +GO + +ALTER DATABASE [purview-sqlmi-db] SET DATE_CORRELATION_OPTIMIZATION OFF +GO + +ALTER DATABASE [purview-sqlmi-db] SET TRUSTWORTHY OFF +GO + +ALTER DATABASE [purview-sqlmi-db] SET ALLOW_SNAPSHOT_ISOLATION OFF +GO + +ALTER DATABASE [purview-sqlmi-db] SET PARAMETERIZATION SIMPLE +GO + +ALTER DATABASE [purview-sqlmi-db] SET READ_COMMITTED_SNAPSHOT OFF +GO + +ALTER DATABASE [purview-sqlmi-db] SET HONOR_BROKER_PRIORITY OFF +GO + +ALTER DATABASE [purview-sqlmi-db] SET RECOVERY FULL +GO + +ALTER DATABASE [purview-sqlmi-db] SET MULTI_USER +GO + +ALTER DATABASE [purview-sqlmi-db] SET PAGE_VERIFY CHECKSUM +GO + +ALTER DATABASE [purview-sqlmi-db] SET DB_CHAINING OFF +GO + +ALTER DATABASE [purview-sqlmi-db] SET FILESTREAM( NON_TRANSACTED_ACCESS = OFF ) +GO + +ALTER DATABASE [purview-sqlmi-db] SET TARGET_RECOVERY_TIME = 60 SECONDS +GO + +ALTER DATABASE [purview-sqlmi-db] SET DELAYED_DURABILITY = DISABLED +GO + +ALTER DATABASE [purview-sqlmi-db] SET ACCELERATED_DATABASE_RECOVERY = OFF +GO + +ALTER DATABASE [purview-sqlmi-db] SET QUERY_STORE = OFF +GO + +ALTER DATABASE [purview-sqlmi-db] SET READ_WRITE +GO \ No newline at end of file diff --git a/examples/ssis/example_data/MovCustomers.csv b/examples/ssis/example_data/MovCustomers.csv new file mode 100644 index 0000000..3b1bd1c --- /dev/null +++ b/examples/ssis/example_data/MovCustomers.csv @@ -0,0 +1,251 @@ +Customer_ID,Last_Name,First_Name,Addr_1,Addr_2,City,State,Zip_Code,Phone_Number,Created_Date,Updated_Date +B09477DC-5DCD-412A-AC13-01606E390264,Kolos,Orsos,268 Tenth Ave,NULL,Washington,DC,21001,5555550561,2017-01-01,2017-07-16 +3B3D46CB-FDB6-494F-82DF-02101A02EDFD,Chandrakanta,Mistry,120 Cherry St,NULL,Austin,TX,75101,5555550033,2017-01-01,2017-11-07 +F7596520-CD3E-490E-A8EF-0349B36EAAE1,Tivadar,Szabo,981 Eighth Ave,NULL,San Diego,CA,98101,5555550997,2017-05-23,2017-10-12 +0A58BEA6-B3BD-4867-9F1D-035046FFB03C,Milada,Radova,654 Sixth Ave,NULL,Minneapolis,MN,58101,5555559498,2017-01-01,2017-09-14 +D05444AD-4558-40BA-A022-047D53E8ACF8,Cristian,Bojin,263 Maple St,NULL,Seattle,WA,92101,5555550330,2017-01-01,2017-10-07 +D199E279-E223-4817-9070-04E1F3285786,Leo,Lamb,732 Maple St,NULL,Portland,OR,85101,5555559179,2017-04-24,2017-06-21 +9C58BA57-CE0A-4613-B08B-0544EF2BCCC8,Miroslav,Horvath,50 Fifth Ave,NULL,Seattle,WA,92101,5555559247,2017-02-02,2017-10-30 +3BA863F6-C9DD-4AC9-A85D-05D6F5F72383,Hattie,Villarreal,808 Fourth Ave,NULL,Seattle,WA,92101,5555550250,2017-01-01,2017-07-20 +550581B7-88E5-4466-A314-06A1CE29ACFD,Saral,Deshpande,145 Dogwood St,NULL,Austin,TX,75101,5555559949,2017-01-01,2017-06-10 +618E11A2-E3F0-4A20-83B9-0712E132D4A4,Stern,Vanderschans,751 Fifth Ave,NULL,Austin,TX,75101,5555559793,2017-05-30,2017-07-28 +1240FB05-6FD2-48A7-BC8A-095144B0F758,Jaidev,Singh,57 Oak St,NULL,New York,NY,12345,5555559846,2017-01-01,2017-09-18 +21CB8446-CC8B-4EC5-9BB3-0AE95F8D3430,Ellen,Peacock,982 Fifth Ave,NULL,Seattle,WA,92101,5555559094,2017-01-01,2017-09-25 +797D5EC7-48F4-49EC-9266-0BB7180F252C,Ethan,Greenwood,802 Ninth Ave,NULL,Portland,OR,85101,5555559457,2017-01-01,2017-11-10 +C9092895-97F0-4120-9B60-0BC59A5BD98C,Ines,Martinez,742 Seventh Ave,NULL,Austin,TX,75101,5555559186,2017-01-01,2017-06-19 +D3745E45-55E6-41D4-A1FD-0C64050165C3,Jackie,Wilder,299 Tenth Ave,NULL,New York,NY,12345,5555559431,2017-01-25,2017-11-18 +14042897-9C47-4E60-962F-0CAFB43F90FE,Bonita,Turner,667 Fourth Ave,NULL,Portland,OR,85101,5555559636,2017-01-01,2017-08-30 +8C6188C5-AEB6-4E3F-8EA1-0D8F718755A5,Lucas,Bowen,552 First Ave,NULL,Portland,OR,85101,5555550414,2017-01-01,2017-08-06 +A38246E8-2680-4D2E-AED2-0EAAE0D69BCC,Julian,Persson,732 Dogwood St,NULL,San Diego,CA,98101,5555550399,2017-02-02,2017-10-19 +EB266CDD-391B-4FDC-877E-1160BCE6CC00,Chaayaadaevi,Mudigonda,581 Eighth Ave,NULL,Seattle,WA,92101,5555550918,2017-01-01,2017-10-30 +8D1ABE81-ABEF-4286-B7F7-11E9EAFCB844,Jay,Rose,759 Tenth Ave,NULL,Portland,OR,85101,5555559455,2017-02-04,2017-11-27 +BB2ACFEF-F4C2-40FE-9376-12D1BE6E27FE,Agne,Lukauskiene,642 Oak St,NULL,Minneapolis,MN,58101,5555559218,2017-01-01,2017-10-23 +28D189D2-C6F3-4775-A58D-12DAE82F50FE,Marianna,Szakaly,997 Seventh Ave,NULL,Minneapolis,MN,58101,5555550206,2017-01-01,2017-08-28 +01B4B2B3-E8C6-445B-8206-137477DE5560,Flynn,Grosse,490 Seventh Ave,NULL,Portland,OR,85101,5555559840,2017-01-01,2017-07-15 +825C6246-ADB6-4908-AC13-1400BEFAEB25,Malinda,Hamilton,925 First Ave,NULL,Austin,TX,75101,5555550851,2017-01-01,2017-07-19 +B781642A-7A76-4315-9476-15FBB72F5317,Liz,Herman,344 Third Ave,NULL,Austin,TX,75101,5555559021,2017-01-01,2017-09-30 +CCAFFD68-9A83-4105-82BC-16488C80DA1E,Demetra,Barberis,783 Tenth Ave,NULL,Minneapolis,MN,58101,5555550656,2017-01-01,2017-06-17 +4E4E6912-D983-4433-836D-1771033E25BD,Rafaela,Rocha,198 Fifth Ave,NULL,Portland,OR,85101,5555550156,2017-01-01,2017-10-24 +FAC4F69E-36FC-4394-9087-180B89B1AC40,Sebo,Soross,573 Palm St,NULL,Austin,TX,75101,5555550480,2017-01-01,2017-09-28 +8AD7BE72-568A-44B7-A512-1A472D0BE270,Nadezhda,Angelovaa,88 Cherry St,NULL,Washington,DC,21001,5555550540,2017-01-01,2017-06-29 +27AD3330-392E-4958-81B2-1B07FA770B47,Charlie,Woods,221 Pine St,NULL,Washington,DC,21001,5555559300,2017-05-10,2017-10-06 +DF8D841B-3EB7-46FA-895B-1B8797A92CB9,Geza,Rozsa,306 Tenth Ave,NULL,Portland,OR,85101,5555550270,2017-05-15,2017-06-21 +C95D4AFF-3AF1-4EB1-8E7B-1D5A7AB211BE,Eden,Ben-Meir,449 Maple St,NULL,Minneapolis,MN,58101,5555550904,2017-01-01,2017-09-17 +6F770F36-02E6-4C77-BE21-1DFDEBADE3D5,Ivana,Solevic,133 Fourth Ave,NULL,New York,NY,12345,5555559937,2017-05-30,2017-08-08 +5D8973DE-1FF3-4641-8B99-2019C9AE87DF,Vicente,Lawhorn,458 First Ave,NULL,Portland,OR,85101,5555550366,2017-02-02,2017-10-21 +DE3F8B5E-E4B4-4695-A894-20CCA5041979,Jay,Rose,723 Palm St,NULL,San Diego,CA,98101,5555550895,2017-01-01,2017-10-30 +FA0039A7-8698-455E-A894-219243FBFD7E,Jun-jie,L?n,493 Fifth Ave,NULL,Portland,OR,85101,5555550023,2017-01-01,2017-11-24 +2581CA70-395C-4582-8A72-24AE7ED59330,Enzo,Onio,240 Oak St,NULL,Portland,OR,85101,5555550934,2017-01-01,2017-08-12 +4F74FAE6-9F6B-4D77-A5C4-26C7D82E4AD4,Karolina,Koz?owska,916 Third Ave,NULL,New York,NY,12345,5555559207,2017-05-16,2017-07-17 +A3F0AAA0-6C49-4DF1-A565-2776848601C1,Olga,Luksic,79 Second Ave,NULL,Seattle,WA,92101,5555550867,2017-01-01,2017-07-20 +01E5C757-DF0C-4941-8E5E-279044086DB5,Donnell,Jankowski,717 Fourth Ave,NULL,Seattle,WA,92101,5555559513,2017-01-01,2017-11-16 +6A0A6105-ACE8-4F35-90B2-291BC9C33B57,Ulysses,Popp,409 Ninth Ave,NULL,San Diego,CA,98101,5555559712,2017-01-01,2017-10-04 +E6942FBD-68FE-476B-B1EF-29D17EF82DC2,Vimala,Cheema,90 Seventh Ave,NULL,Austin,TX,75101,5555550691,2017-01-01,2017-08-26 +16E9F859-3EEC-41E6-AD94-2ABA593F0D5D,Stefan,Fryer,917 Sixth Ave,NULL,Seattle,WA,92101,5555559327,2017-01-01,2017-08-16 +DB8CCAF2-E42F-457A-BBA0-2D3EE1826261,Greta,Leonard,658 Ninth Ave,NULL,Austin,TX,75101,5555559964,2017-01-01,2017-10-24 +0E4897ED-2271-49B8-98DA-2EAD15F23850,Amelie,Koch,449 Sixth Ave,NULL,Minneapolis,MN,58101,5555559571,2017-01-01,2017-10-28 +25127C2C-4874-42BE-9567-2EBF26DB9ECD,Natasza,Michalska,709 Fifth Ave,NULL,Portland,OR,85101,5555559433,2017-01-01,2017-08-27 +D61E6E18-A323-4438-8FFC-3103A6AA86F4,Vern,Piazza,258 Tenth Ave,NULL,Austin,TX,75101,5555559797,2017-01-01,2017-06-07 +1BE58839-D0B4-44D2-A7A8-318221C99F6F,Tatjana,Ivanovic,561 Fifth Ave,NULL,Portland,OR,85101,5555559938,2017-01-01,2017-11-04 +23744E24-C344-4641-A0ED-326507C89C94,Min,Kuo,561 First Ave,NULL,New York,NY,12345,5555550138,2017-01-01,2017-09-22 +2E52091F-049D-49B9-A3DE-335E1432C873,Angela,Dominguez,556 Elm St,NULL,New York,NY,12345,5555550176,2017-01-01,2017-08-20 +FE93F04F-AEC5-460F-B299-33F945A2C78A,Amelia,Martel,512 Palm St,NULL,Austin,TX,75101,5555559699,2017-01-01,2017-09-20 +34BFAE92-C57A-4C66-9D8F-349B391E202B,Dilshad,Rabiel,104 First Ave,NULL,Minneapolis,MN,58101,5555550053,2017-01-01,2017-09-03 +FB32BB33-BB45-4635-9601-34E812DB8906,Biljana,Petrovic,469 Tenth Ave,NULL,San Diego,CA,98101,5555559161,2017-01-01,2017-09-10 +C1CB6720-C287-4EFA-8607-354F8652A8DE,Maureen,Couture,486 Cherry St,NULL,New York,NY,12345,5555550085,2017-01-01,2017-06-30 +5E117FD6-6077-4E76-BA88-35D7F1D50B6B,Rato,Jovanovic,347 Tenth Ave,NULL,Seattle,WA,92101,5555550533,2017-01-01,2017-06-12 +68F75F70-10DF-495B-8043-365C37497AC1,Frank,Smits,109 Elm St,NULL,Austin,TX,75101,5555559611,2017-01-19,2017-08-19 +F41A357C-44F1-4F0A-BAAD-3660D1774D61,Nebojsa,Haravan,238 Fifth Ave,NULL,Washington,DC,21001,5555559599,2017-01-01,2017-08-25 +C227B60B-973B-4AC6-981F-38CE87E5F4C6,Agne,Vengryte,772 Dogwood St,NULL,Austin,TX,75101,5555559882,2017-03-30,2017-08-18 +32B40FAD-DD37-4565-AB81-3A21FD39FD2F,Shou,Kaneko,47 Fourth Ave,NULL,Minneapolis,MN,58101,5555550964,2017-01-01,2017-08-04 +5FA8CB33-6C30-4425-914B-3A25E8A73B52,Carla,Goncalves,803 Maple St,NULL,Washington,DC,21001,5555559558,2017-01-01,2017-09-08 +D1F904C9-9862-4378-A159-3AD5CFC3A3E3,Nerea,Estevez,144 Cherry St,NULL,Austin,TX,75101,5555550932,2017-01-20,2017-10-05 +4656DD9F-3993-424A-B14A-3B4259552035,Eliza,Wysocka,2 Oak St,NULL,Portland,OR,85101,5555550210,2017-01-01,2017-08-16 +FBC9F5BE-7AB8-4C09-BB7E-3C85EAEE7E5A,Saika,Aoki,712 Elm St,NULL,Seattle,WA,92101,5555550192,2017-01-01,2017-06-21 +F1452EF4-1889-4E25-8EEF-3D10C0599428,Myra,Alvarez,773 Second Ave,NULL,Seattle,WA,92101,5555559759,2017-01-01,2017-09-23 +3B72270D-245E-4059-AA7C-3D3A90ADF4FF,Kim Cuc,Thieu,60 Dogwood St,NULL,Seattle,WA,92101,5555559393,2017-03-03,2017-07-20 +91AB450F-09A1-4611-8F26-3D861C3304B0,Nicolo,Gallo,565 Palm St,NULL,New York,NY,12345,5555559006,2017-01-01,2017-08-13 +48E8EC1C-5287-4D18-9D56-3EFFCE4B77A9,Laverne,Fischer,768 Pine St,NULL,Seattle,WA,92101,5555559988,2017-04-07,2017-09-15 +E70ECD88-F81E-482A-8F52-3F58768C99F1,Arnost,Pajer,962 Tenth Ave,NULL,Minneapolis,MN,58101,5555559434,2017-01-16,2017-07-16 +621893B7-4C54-4A14-815B-40C2E46B5EE6,Heidi,Pichler,532 Third Ave,NULL,Seattle,WA,92101,5555559731,2017-02-18,2017-08-19 +B80F0CEE-0099-476E-B5E5-40CFF39A3640,Kazuya,Sakai,370 Fourth Ave,NULL,San Diego,CA,98101,5555559825,2017-01-01,2017-07-08 +BEDE2931-70FA-4EF9-AB82-41E7527FE65B,Pearlie,Peters,797 Fifth Ave,NULL,Seattle,WA,92101,5555550547,2017-01-01,2017-09-08 +87D5880B-0030-4580-AE85-432CE3F17263,Elisabeta,Buscan,254 Cherry St,NULL,Minneapolis,MN,58101,5555559429,2017-05-28,2017-11-16 +19D64913-275C-49B1-AA7D-43A4C68DCDB1,Matyas,Bohuslav,123 Ninth Ave,NULL,Washington,DC,21001,5555559262,2017-02-26,2017-11-27 +9864AA6C-2FE9-414E-9BF5-43F81DBEA47D,Szabolcs,Lovas,29 Oak St,NULL,Portland,OR,85101,5555550624,2017-01-01,2017-10-24 +D666ACCF-F56C-4597-8053-4432782CCB99,Shakshavali,Pamulaparthi,779 Dogwood St,NULL,New York,NY,12345,5555550855,2017-01-01,2017-06-08 +D6A63C46-FB25-44D5-A530-4742A997A17F,Paula,Teixeira,823 Fifth Ave,NULL,Minneapolis,MN,58101,5555550421,2017-01-01,2017-06-26 +D5F16015-3F57-4076-A8EF-4793BA1016EB,Elijah,Dionne,640 Ninth Ave,NULL,Washington,DC,21001,5555550600,2017-01-01,2017-09-12 +CD1F11AF-0656-4A20-AF06-492549AFE0AF,Odon,Mate,924 Tenth Ave,NULL,Austin,TX,75101,5555559544,2017-03-31,2017-10-04 +44498243-7D52-4A72-8168-49E495AE319C,Slavoljub,Komljenovic,619 Elm St,NULL,Portland,OR,85101,5555550410,2017-02-18,2017-08-13 +D6E23090-6E6F-49CA-8283-4B35FB3BAEDF,Abhishek,Gadiyaram,940 Pine St,NULL,Minneapolis,MN,58101,5555559909,2017-01-01,2017-10-12 +4F4C2A71-43E1-4CCF-9119-4CE81A541FFD,Chin-Sun,Oh,522 Maple St,NULL,New York,NY,12345,5555550806,2017-01-01,2017-08-16 +0A0C072B-FF1E-4E61-86AE-4FC8E5FAB9C7,Judit,Foldy,851 Sixth Ave,NULL,Austin,TX,75101,5555559367,2017-01-24,2017-07-10 +D07919EB-5863-44A8-9CAC-4FF8242D2345,Beulah,Fleming,320 Ninth Ave,NULL,Washington,DC,21001,5555559597,2017-01-01,2017-07-06 +2AA56ECD-F4C6-4330-A984-50A58CA95764,Kristin,Kasesalu,492 Ninth Ave,NULL,Washington,DC,21001,5555559059,2017-01-01,2017-11-03 +B1EB714B-95A6-4769-8B52-516F9FAD8F30,Anda,Liepa,617 Oak St,NULL,Portland,OR,85101,5555550838,2017-01-01,2017-11-22 +A384AD93-26A4-4ED1-89B5-526471BB4914,Onele,Skujyte,885 Dogwood St,NULL,Washington,DC,21001,5555559856,2017-01-01,2017-07-27 +D42EBFE4-37D9-4814-A01F-5301103D258A,Maxime,Walravens,904 Seventh Ave,NULL,Seattle,WA,92101,5555559334,2017-01-01,2017-08-27 +1E9FF997-8255-43B7-840C-53D1290C2F16,Jacopo,Lorenzo,922 Elm St,NULL,Minneapolis,MN,58101,5555559839,2017-01-01,2017-09-08 +18ED2E51-D127-43C1-9E1E-548169144CBA,Jerica,Slosar,819 Third Ave,NULL,New York,NY,12345,5555559758,2017-01-01,2017-06-25 +06A9670D-CD24-403F-BA33-54BBC2ADBE70,Rato,Jovanovic,697 First Ave,NULL,Portland,OR,85101,5555559889,2017-01-01,2017-07-23 +0CB25692-6FD8-4E6B-BB4C-55D8A8C33BCE,Yua,Sasaki,960 Palm St,NULL,San Diego,CA,98101,5555559198,2017-01-01,2017-07-17 +06BC8405-A121-488B-856C-5655FCEF62BB,Leonardo,Buccho,342 Third Ave,NULL,Washington,DC,21001,5555559016,2017-01-01,2017-11-23 +9DD340A8-1019-4206-A6BB-56EE546137C1,Jaclyn,Daniels,46 Seventh Ave,NULL,New York,NY,12345,5555559901,2017-04-14,2017-10-02 +BCAE0982-5E12-4675-A82B-573BD1BFF34C,Malinda,De Crom,270 Oak St,NULL,Portland,OR,85101,5555559159,2017-01-01,2017-09-02 +3062B86E-66C1-4A23-BFE5-5753B180DF88,Elise,Morales,604 Tenth Ave,NULL,Seattle,WA,92101,5555559524,2017-01-01,2017-09-13 +128CD655-A036-4764-82D7-58755FFAEFD4,Kelemen,Bernath,493 Fifth Ave,NULL,Austin,TX,75101,5555550231,2017-04-04,2017-08-28 +F8B369DE-73A9-454C-9C89-588AE10C46B9,Kim,Dang,673 Eighth Ave,NULL,Seattle,WA,92101,5555559560,2017-01-01,2017-09-25 +6798EEE0-E899-4301-B8D1-59F3B40B63D1,Yelyzaveta,Oliynyk,991 Second Ave,NULL,San Diego,CA,98101,5555559120,2017-01-01,2017-07-31 +B5F25654-0F93-4082-9416-5A78EC081071,Ellen,Gardiner,164 Tenth Ave,NULL,San Diego,CA,98101,5555550189,2017-02-04,2017-06-10 +1D6526FF-6FE2-4D54-B88A-5AD1AD27007A,Min-ji,Paik,479 Sixth Ave,NULL,Portland,OR,85101,5555559204,2017-01-01,2017-08-29 +3BB19B3F-7967-479E-A362-5B579EDD86F3,Pauline,Simon,737 Third Ave,NULL,Minneapolis,MN,58101,5555550954,2017-01-01,2017-06-30 +BDB92A16-B428-4779-8DF3-5BAC45247DF2,Ruby,Kumm,114 Maple St,NULL,Austin,TX,75101,5555559983,2017-02-09,2017-06-07 +35E060C7-F3C8-4D18-8140-5CF80EF34CAC,Christos,Ioannou,260 Tenth Ave,NULL,Seattle,WA,92101,5555559297,2017-01-01,2017-11-06 +319BD965-AA15-4017-87C1-5DF1ED690E3E,Branimir,Lechkov,286 Fourth Ave,NULL,Minneapolis,MN,58101,5555559935,2017-01-01,2017-06-29 +58A6E6E5-B4CD-4A19-B4C0-5ECB1627038A,Vicky,Barlow,206 Eighth Ave,NULL,Austin,TX,75101,5555559499,2017-01-01,2017-11-03 +3291F9D7-B7CF-4D89-A634-61B9F231B8FA,Hao,Cao,36 Sixth Ave,NULL,Washington,DC,21001,5555559600,2017-01-01,2017-09-27 +0768152B-2E5A-443E-88E2-62434E7EA09E,Hanne,Nissen,177 Ninth Ave,NULL,Washington,DC,21001,5555559902,2017-01-01,2017-07-08 +47B9E9AF-76BC-488A-ACA2-627D30C19ADE,Nedelya,Hristova,247 Fourth Ave,NULL,New York,NY,12345,5555559885,2017-01-01,2017-09-28 +365E1B5D-4A17-4C2A-8B18-63EC9D3CD0C8,Nehorai,Shapiro,372 Eighth Ave,NULL,San Diego,CA,98101,5555559030,2017-05-18,2017-10-21 +0690DB2E-D4EE-4B73-8E3D-6694B1330B1B,Carlos,McKay,576 Fourth Ave,NULL,San Diego,CA,98101,5555550069,2017-01-01,2017-08-31 +CA15B5B8-C8E2-4467-8C61-66E703B5C482,Chung-Ho,Ju,852 Eighth Ave,NULL,Austin,TX,75101,5555559804,2017-01-01,2017-10-30 +792BC953-83D5-45DC-98CA-673C05F6D589,Marcel,Levesque,874 Maple St,NULL,San Diego,CA,98101,5555559987,2017-01-01,2017-09-23 +04C573A8-DABF-4B98-A8A0-676D2D323AEB,Judith,Tate,47 Fourth Ave,NULL,Seattle,WA,92101,5555559476,2017-01-01,2017-11-05 +A64DCA71-D8CC-4F9F-AC07-68D5060DDACA,Miguel Angel,Montoya,597 Pine St,NULL,Seattle,WA,92101,5555559781,2017-01-01,2017-07-04 +23D28B5F-EA9A-440A-BDEA-6929D7915838,Sebastian,Wintle,902 Oak St,NULL,San Diego,CA,98101,5555550450,2017-01-01,2017-09-26 +0DD10694-E970-4C2E-9C60-699C9B913FFE,Melani,Escobedo,63 Fifth Ave,NULL,Portland,OR,85101,5555559376,2017-01-01,2017-08-26 +1CF2798B-A705-4B5D-A0EE-69F0D258E67B,Sergio,Espinoza,76 Oak St,NULL,Portland,OR,85101,5555550455,2017-01-07,2017-10-13 +4D25A206-1465-4617-A3CF-6C21BD28B6A9,Sun,Xie,363 Oak St,NULL,Minneapolis,MN,58101,5555559153,2017-01-01,2017-10-12 +8A6D7861-7BEF-4F28-8654-6E430F447EC0,Megan,Wood,884 Second Ave,NULL,Austin,TX,75101,5555559229,2017-01-01,2017-09-18 +8173CECE-1B48-4609-B895-6E99261C8875,Denis,Tatarescu,247 Seventh Ave,NULL,Austin,TX,75101,5555559926,2017-04-06,2017-08-11 +156B955F-144C-464C-AEA0-71E054DFA891,Mauri,Lepisto,483 Cherry St,NULL,Portland,OR,85101,5555559777,2017-01-01,2017-08-25 +5F16F978-5791-44B7-9AC4-724298B35E5B,Helen,Frandsen,173 Tenth Ave,NULL,Washington,DC,21001,5555550539,2017-01-01,2017-11-04 +A6FF3C5E-FEE0-4FF3-836E-72C377C9B737,Brittney,Henderson,409 Second Ave,NULL,San Diego,CA,98101,5555550038,2017-04-02,2017-06-27 +D9AE66B2-F4B9-40BB-B0BD-74769C47EBD9,Isabelle,Charlton,716 Eighth Ave,NULL,Austin,TX,75101,5555559645,2017-01-01,2017-10-09 +65881A74-9177-4E27-BBE2-7527A4D66C96,Damyanti,Nandamuri,125 Dogwood St,NULL,Washington,DC,21001,5555559263,2017-05-17,2017-07-19 +0A9E5397-9B2B-4F7A-9A27-76DD03FA6FF9,Marco,Viera,628 Fifth Ave,NULL,San Diego,CA,98101,5555550396,2017-01-01,2017-08-15 +75C820EA-C108-4278-B527-78F59726CE00,Clifton,Deyoung,628 Fifth Ave,NULL,New York,NY,12345,5555550526,2017-01-01,2017-09-28 +48096551-4719-4317-93D7-799753715739,Elaine,Hardy,792 Tenth Ave,NULL,Portland,OR,85101,5555550809,2017-05-07,2017-10-05 +BBBB2EC8-88E1-4803-9E13-7A80613EA5D1,Kaur,Gowda,90 Dogwood St,NULL,New York,NY,12345,5555550726,2017-04-15,2017-07-23 +E094B10F-E12A-4101-97B7-7C4419D5F706,Shai,Lando,919 Palm St,NULL,Seattle,WA,92101,5555559570,2017-01-01,2017-07-22 +AFB83412-CC75-43D9-BBA9-7C8F99030CC7,Min-ji,Paik,206 Maple St,NULL,Washington,DC,21001,5555550760,2017-01-01,2017-09-28 +FCEC662A-1F7C-48FB-B1B1-7D4E5A2C8010,Magda,Codreanu,733 Fourth Ave,NULL,Minneapolis,MN,58101,5555559807,2017-04-17,2017-09-03 +32D9D5A3-773C-4C70-838D-7D7FA12A46DD,Hiroto,Nakayama,227 Fifth Ave,NULL,New York,NY,12345,5555559102,2017-01-01,2017-08-24 +B96279EA-1EFE-4FA6-8E02-7E75F81A25CE,Bao,Liu,983 Dogwood St,NULL,Seattle,WA,92101,5555559003,2017-01-01,2017-06-20 +C5A86C80-8F16-4543-BE9E-7E8158DFDBA4,Sammy,Saxon,2 Oak St,NULL,Portland,OR,85101,5555559959,2017-01-01,2017-06-15 +5D05837D-826F-4C6F-99A5-7F7919337767,Antonella,Madera,970 Dogwood St,NULL,Seattle,WA,92101,5555550397,2017-01-01,2017-09-29 +6D040238-5296-48E5-9B59-802B3ABF5FA5,Geza,Csorba,915 Cherry St,NULL,Washington,DC,21001,5555559110,2017-05-27,2017-09-09 +11C580AC-B9F4-417B-972B-82664A738232,Lata,Kotadia,526 First Ave,NULL,Austin,TX,75101,5555559365,2017-03-17,2017-09-12 +849155C3-992C-4A81-BE98-83F36E139367,Radko,Konstantinov,105 Seventh Ave,NULL,Austin,TX,75101,5555550334,2017-01-28,2017-10-26 +BB80E867-F164-44CE-B737-8488A3BE510B,Skye,Holland,507 Tenth Ave,NULL,New York,NY,12345,5555559177,2017-04-10,2017-08-19 +FABC9F96-98D1-4381-B071-8797D4D2AEC7,Rossitsa,Bojidarova,422 Second Ave,NULL,Austin,TX,75101,5555550463,2017-01-01,2017-09-18 +B8B9CE77-91FC-4DFE-B691-89C96E6CFE47,Shai,Lando,545 First Ave,NULL,Portland,OR,85101,5555550021,2017-01-01,2017-06-15 +04D33548-570B-4F72-AE52-8AB4FC3D7300,Rajan,Mukherjee,668 Oak St,NULL,New York,NY,12345,5555559277,2017-03-10,2017-09-04 +4B302B91-A1D1-4C7A-BC98-8BBB37339124,Lukas,Petrzelka,167 Dogwood St,NULL,Portland,OR,85101,5555559255,2017-01-01,2017-10-14 +311FE866-5C10-4DA4-B3D7-8C016152145D,Hanna,Kangur,292 Palm St,NULL,Minneapolis,MN,58101,5555550098,2017-03-10,2017-11-02 +316F0E28-AE67-4D10-8060-8C1FA39940C3,Leonas,Rubis,557 Ninth Ave,NULL,New York,NY,12345,5555550812,2017-03-04,2017-10-21 +9165D6A6-3BCC-4792-94EC-8DE4B9EA470C,Agne,Terentjeva,396 Dogwood St,NULL,Washington,DC,21001,5555550762,2017-02-17,2017-11-15 +B70B1F3B-190A-4B92-8C4E-8FB372590ED5,Milena,Sofiyanska,557 Dogwood St,NULL,Washington,DC,21001,5555550248,2017-01-01,2017-07-06 +D0257B82-E1EC-4011-A1C7-902736D1EBAF,Vimala,Cheema,640 Ninth Ave,NULL,Washington,DC,21001,5555559678,2017-01-01,2017-07-08 +A64392E4-9E16-463A-A9E5-91917F304056,Roberto,Bianchi,988 Tenth Ave,NULL,Austin,TX,75101,5555550293,2017-01-01,2017-07-11 +BD917FBB-565F-4006-BD32-923AEF5B2F37,Marijonas,Danis,402 Cherry St,NULL,Seattle,WA,92101,5555559224,2017-01-01,2017-11-27 +3AC5CF3A-ECB7-41E9-98FC-937D24F76D55,Yua,Sasaki,133 Fourth Ave,NULL,Washington,DC,21001,5555550630,2017-01-01,2017-10-27 +C89B0561-6A5B-43B5-970C-95092F98E54A,Ali,Dixon,114 First Ave,NULL,Portland,OR,85101,5555559236,2017-02-14,2017-07-02 +3FC5A08A-F8EB-43E3-BBD1-9723DE2C17E4,Dragan,Jovanovic,171 Third Ave,NULL,New York,NY,12345,5555559576,2017-01-01,2017-08-17 +1091ABAD-5702-4061-8D3C-97E1DE2EC15D,Ibrahim,Dolmovic,99 Maple St,NULL,Minneapolis,MN,58101,5555550080,2017-01-01,2017-08-04 +B3100379-233A-4493-9378-982CBEF1F17E,Asiya,Ospankyzy,1 Fourth Ave,NULL,Washington,DC,21001,5555559670,2017-03-28,2017-07-14 +B5CD900B-2EAF-4FD7-B865-9833E97BF0CC,Ergo,Vitsut,954 Dogwood St,NULL,Portland,OR,85101,5555550290,2017-01-01,2017-06-01 +23EA946E-B0B6-4A99-9476-99EA69BE55F5,Mollie,Marsh,445 First Ave,NULL,Seattle,WA,92101,5555550415,2017-01-01,2017-11-16 +0185BCF5-4098-4065-9C90-9B84E822A9A7,Jurgis,Backus,807 Eighth Ave,NULL,San Diego,CA,98101,5555559624,2017-01-01,2017-09-06 +9CA21C61-1A4F-435C-94AE-9BCC73E388A5,Stanko,Konstantinov,743 Maple St,NULL,San Diego,CA,98101,5555550471,2017-01-01,2017-10-11 +E430AFB5-3AE0-44DB-A0AF-9CA11AE80125,Augustinas,Simkus,720 Sixth Ave,NULL,Austin,TX,75101,5555559112,2017-01-01,2017-08-12 +33C81844-09BF-48F4-AAF9-9D2E5AD7CCE4,Leo,Baxter,288 Fourth Ave,NULL,San Diego,CA,98101,5555559274,2017-01-01,2017-07-23 +77F326C6-DFD3-46BE-A9DD-9D9ED16CA368,Russel,Breland,965 Seventh Ave,NULL,Seattle,WA,92101,5555550943,2017-01-01,2017-08-14 +05AD7548-163A-44F1-83BF-9E046C6A833F,Dayaananda,Malladi,777 Ninth Ave,NULL,Minneapolis,MN,58101,5555550099,2017-03-17,2017-11-04 +2CEC4946-FF6F-415B-B957-9E6BB77A52B0,Zorica,Nikolic,306 Tenth Ave,NULL,Washington,DC,21001,5555559519,2017-03-14,2017-09-09 +482F0BD6-C167-4B30-8EBF-9ED1604402D0,Aleksandra,Zirne,548 Elm St,NULL,Seattle,WA,92101,5555559727,2017-01-01,2017-10-26 +D7125DBE-F242-4D62-B817-9FF04F463420,Laxmi,Das,714 Third Ave,NULL,San Diego,CA,98101,5555550226,2017-01-01,2017-06-03 +21158F3A-2572-40DE-B96C-A145FBC4A7F0,Markus,Lippmaa,561 Fifth Ave,NULL,Minneapolis,MN,58101,5555550791,2017-01-01,2017-08-09 +2FFA319D-B82D-48CE-83C0-A2755E996967,Martins,Kalejs,524 Dogwood St,NULL,Seattle,WA,92101,5555550459,2017-01-01,2017-07-15 +D56AC638-70E1-42C2-8B47-A37B812B1DCA,Young-Tae,Park,878 Second Ave,NULL,San Diego,CA,98101,5555559572,2017-01-01,2017-07-15 +18A6E423-86B5-4DFA-AB01-A3BEDBA18989,Ernest,Vanburen,166 Tenth Ave,NULL,Washington,DC,21001,5555550060,2017-01-01,2017-10-09 +308CE9CF-EF8D-4ECD-8570-A5EF9C6A1656,Tian,Ren,717 Elm St,NULL,Minneapolis,MN,58101,5555559164,2017-01-01,2017-11-02 +4FEC161A-90D0-494C-B0CA-AA08B10D2161,Hsiu-ching,Liu,547 Fourth Ave,NULL,Seattle,WA,92101,5555550166,2017-01-01,2017-10-23 +D5558AAB-BC2E-47A8-B20C-AAF7F6E5EAE1,Deepa,Bollimunta,113 Sixth Ave,NULL,Austin,TX,75101,5555559375,2017-01-01,2017-10-01 +0F51E7AD-60D3-4776-A742-AC70DB33D5B3,Waldemar,Sedlar,145 Third Ave,NULL,Austin,TX,75101,5555559803,2017-01-23,2017-07-30 +A6F4CAE0-9DD8-4D5A-8E70-AD6D116C4352,Yeow,Guo,417 Second Ave,NULL,Portland,OR,85101,5555559989,2017-02-03,2017-10-08 +CE8F0C65-5594-442F-A6EC-B066C2A87E7C,Willem,Ellenbroek,852 Third Ave,NULL,Austin,TX,75101,5555559978,2017-01-01,2017-08-06 +90034F4D-A80E-46F2-A803-B23BAED4A124,Ingrida,Kundzina,647 Maple St,NULL,San Diego,CA,98101,5555559231,2017-05-28,2017-07-14 +01B179D9-2568-457C-B219-B2CF75328DA3,Winifred,Jensen,756 Seventh Ave,NULL,New York,NY,12345,5555559967,2017-04-14,2017-08-13 +ED9B1F3B-B03C-4523-BF79-B480A007DAE6,Lorand,Kovacs,628 Fifth Ave,NULL,San Diego,CA,98101,5555559551,2017-02-17,2017-07-26 +248B5F8A-ABF4-4957-ADEB-B4B98A6C03E9,Rozalija,Grybauskaite,409 Dogwood St,NULL,Minneapolis,MN,58101,5555559809,2017-01-01,2017-11-06 +D213978D-0906-4BC6-B44A-B5E09914D652,Suraj,Ogra,831 Fourth Ave,NULL,Austin,TX,75101,5555559250,2017-01-01,2017-08-12 +43ED1B51-3C5F-492B-BAB8-B6A41112C7E2,Oscar,Delgadillo,338 First Ave,NULL,New York,NY,12345,5555559705,2017-01-01,2017-09-04 +C02CA2BF-06C2-454C-8BEB-B9867388716C,Bhagavati,Sonti,841 Ninth Ave,NULL,New York,NY,12345,5555559200,2017-01-01,2017-07-25 +4367C963-F7BA-490A-AFD4-BA73D2F34E76,Zivah,Schlossberg,141 Palm St,NULL,San Diego,CA,98101,5555550139,2017-04-17,2017-07-07 +D8229705-C7A7-42EE-B1D4-BCD9BD548606,Anna,Liepa,465 Elm St,NULL,Washington,DC,21001,5555550961,2017-01-01,2017-11-12 +0AAE3A3C-8AA7-4411-85FE-BEE55C59F562,Padavina,Savicevic,975 Eighth Ave,NULL,Washington,DC,21001,5555559033,2017-01-01,2017-06-08 +18E2BAE0-4997-4A48-A4CF-BF1D3E1C37DB,Sakura,Takahashi,503 Dogwood St,NULL,San Diego,CA,98101,5555559244,2017-01-01,2017-10-22 +B14EA582-B33C-4143-8A4C-BFF86096329A,Amelia,Butts,597 Pine St,NULL,New York,NY,12345,5555550804,2017-01-01,2017-06-06 +FB4BBE3C-20E5-4DF9-9C3A-C102AA234997,Eliza,Wysocka,720 Eighth Ave,NULL,San Diego,CA,98101,5555559395,2017-04-16,2017-08-21 +3C52474B-041A-4E91-8C91-C10C4FA409E3,Mudar,Markovic,619 Seventh Ave,NULL,Portland,OR,85101,5555559947,2017-01-01,2017-07-02 +BCBD8C61-304F-4F6F-B922-C11400365BDE,Emilia,Fisher,445 Tenth Ave,NULL,Seattle,WA,92101,5555559330,2017-01-01,2017-06-06 +3C17F7AB-F6C9-4068-A982-C2228631F7FE,Mirela,Bacic,38 Seventh Ave,NULL,New York,NY,12345,5555559850,2017-01-01,2017-09-04 +7CA0200A-CE8D-45FE-9143-C2B35DEAC47B,Nemanja,Popovic,755 Seventh Ave,NULL,Portland,OR,85101,5555559899,2017-01-01,2017-08-27 +6BAF4315-8CCC-4CAE-89AD-C322A84BE39B,Ella,Goderich,380 Eighth Ave,NULL,Austin,TX,75101,5555559065,2017-01-01,2017-06-12 +6A4713D2-39E0-484D-B342-C51C012B7A73,Ting,Kao,961 Pine St,NULL,Portland,OR,85101,5555559696,2017-01-23,2017-07-27 +7161D1FA-C038-4B08-BC35-C5BC02E85517,Miyu,Yamamoto,737 Ninth Ave,NULL,Portland,OR,85101,5555550265,2017-01-09,2017-07-22 +E49CC54D-73EE-4BA8-9EF1-C5E97EC861EA,Ruby,Sawyer,488 Pine St,NULL,Minneapolis,MN,58101,5555550953,2017-01-24,2017-09-12 +B4AA6410-7478-4008-97D1-C63DAE7FE4E5,Andrius,Slezas,778 Seventh Ave,NULL,New York,NY,12345,5555559823,2017-05-18,2017-11-11 +8561B80B-BAE9-469C-BA19-C6C76A2100C6,Phyllis,Hubbard,214 Elm St,NULL,Austin,TX,75101,5555559567,2017-05-04,2017-09-28 +F7F72598-2559-4703-B56C-C8488F8DC8DF,Zhelyazko,Angelov,106 Cherry St,NULL,Minneapolis,MN,58101,5555550084,2017-01-01,2017-06-28 +706CCEE5-4E7B-42F7-953B-C86DF761586E,Carlos,McKay,93 Pine St,NULL,Portland,OR,85101,5555559985,2017-01-01,2017-09-04 +3955985F-77AB-4FA8-B34E-C87F72BE52CB,Mirella,Huber,56 Sixth Ave,NULL,Austin,TX,75101,5555559914,2017-01-01,2017-10-11 +0B42F14A-A4AE-409D-A4FE-CA6C081DEE96,Elize,Loohuis,189 Eighth Ave,NULL,Portland,OR,85101,5555559653,2017-02-14,2017-10-01 +BC1BE23A-1B9E-433E-B244-CBF4501B71B6,Kaur,Gowda,959 Fourth Ave,NULL,Seattle,WA,92101,5555559824,2017-01-01,2017-06-01 +314D398A-551E-46C8-80DF-CD600FC1B760,Sophie,Benson,489 Fifth Ave,NULL,Portland,OR,85101,5555559756,2017-01-01,2017-06-16 +0F6EA07E-653C-4949-A153-CDF61C0CBBE9,Igor,Zeman,574 Ninth Ave,NULL,Washington,DC,21001,5555559635,2017-01-01,2017-07-01 +803B8043-D975-42FF-BC1A-CE99DB1844AD,Suzanne,Christian,90 Seventh Ave,NULL,Minneapolis,MN,58101,5555559045,2017-01-01,2017-09-21 +F55C341B-17CD-46BE-95C5-CFA48CD3D637,Brock,Fredrickson,656 Oak St,NULL,San Diego,CA,98101,5555559301,2017-03-28,2017-07-17 +AE6F647C-DCC8-4681-8C22-D0B493A23B43,Janice,Gilliam,34 Ninth Ave,NULL,Washington,DC,21001,5555559079,2017-01-01,2017-08-06 +56C4AE1F-FF32-4D57-8147-D0D8EA5E6286,Ana,Cardoso,641 Sixth Ave,NULL,Austin,TX,75101,5555550850,2017-01-01,2017-10-03 +84207B0B-9A67-4F48-A1D8-D3A7CE6AB5BD,Isa,Loohuis,518 Pine St,NULL,New York,NY,12345,5555550458,2017-01-01,2017-09-25 +D0CC41AC-33E9-44DB-8016-D43786727BC7,Desislav,Yankov,532 Third Ave,NULL,Seattle,WA,92101,5555550059,2017-01-01,2017-10-21 +F91C97E2-D2A9-437E-B680-D9729F0A69E2,Brigitta,Szarka,41 Ninth Ave,NULL,Portland,OR,85101,5555559272,2017-01-01,2017-08-07 +AA7AFFDE-B309-449D-AA74-D983A04DC043,Lina,Panicucci,805 Sixth Ave,NULL,New York,NY,12345,5555559165,2017-01-01,2017-10-25 +000367E9-81DA-45D3-9391-DB4A3143C76E,Justine,Cantu,954 Dogwood St,NULL,Washington,DC,21001,5555559128,2017-01-01,2017-08-01 +BF853613-E6E6-4180-A596-DDB6195ECB52,Lucas,Bowen,395 Fourth Ave,NULL,Portland,OR,85101,5555559903,2017-01-01,2017-06-21 +DC228174-2702-4E8B-81A6-DDB685FFE24E,Aidan,Moresby,828 First Ave,NULL,Washington,DC,21001,5555550944,2017-03-11,2017-07-28 +CBAD2F37-5BC8-4936-876C-E0569F8763E3,Karolina,Koz?owska,254 Ninth Ave,NULL,Portland,OR,85101,5555550285,2017-05-11,2017-11-23 +E27E87A0-9D3A-48C7-A4A4-E2F9D3410228,Marco,Costa,656 Tenth Ave,NULL,Austin,TX,75101,5555550089,2017-01-01,2017-06-29 +44720A85-D9D9-4816-B338-E4DA5688DED5,Nadija,Bradas,965 Seventh Ave,NULL,Portland,OR,85101,5555559933,2017-02-07,2017-09-29 +3922DA09-D0D5-49FD-934A-E5886B621D2E,Tam,Do,335 Cherry St,NULL,New York,NY,12345,5555550735,2017-04-15,2017-06-21 +31E27D83-EA14-4682-951A-EB216CBA7611,Jarmila,Blazkova,660 Dogwood St,NULL,Washington,DC,21001,5555550075,2017-02-14,2017-08-03 +BC4592D3-197D-4915-BE01-EBC93A32AD83,Cai,Xue,725 Ninth Ave,NULL,Portland,OR,85101,5555550344,2017-01-01,2017-07-23 +A9ED4649-DAD5-4DE0-9DBE-EBF71872E959,Viktorija,Stankeviciene,717 Fourth Ave,NULL,San Diego,CA,98101,5555550339,2017-01-01,2017-08-02 +236AF0F9-3123-46BE-BB70-EC3D03AFF147,Bohdan,Mirniy,317 Ninth Ave,NULL,San Diego,CA,98101,5555559561,2017-01-01,2017-10-01 +8E05C086-0F08-4DA0-8DCF-EDC290ECB043,Katalin,Voros,16 Fifth Ave,NULL,Minneapolis,MN,58101,5555550177,2017-01-01,2017-10-22 +F7E382C7-A2EF-4F3A-909F-EE5C4FCE5F42,Helena,Costa,98 Sixth Ave,NULL,Austin,TX,75101,5555559690,2017-01-01,2017-10-17 +33F9CEC9-3046-4DA3-A381-EE6626945BCB,Willa,Dawson,75 Fifth Ave,NULL,San Diego,CA,98101,5555559818,2017-01-01,2017-07-16 +6F035883-3AC6-43A0-B466-EF2BF4B04798,Hsiung,Yuan,308 Third Ave,NULL,Minneapolis,MN,58101,5555559548,2017-01-01,2017-09-14 +010A80AF-73D7-4AB6-A164-F0D62F5E3CCA,Ali,Espino,416 Fifth Ave,NULL,Austin,TX,75101,5555550718,2017-01-01,2017-08-28 +10BCE7DE-E89C-49B2-A9D3-F10066BAF636,Gedeminas,Shimkus,131 Tenth Ave,NULL,Washington,DC,21001,5555559545,2017-01-01,2017-08-24 +B15FF7F9-11D7-48D8-9DA9-F1083D47318A,Sung-Hwan,Oh,145 Sixth Ave,NULL,New York,NY,12345,5555550319,2017-01-01,2017-08-04 +B83DE811-F2CE-4941-857F-F1242F16961A,Yua,Maeda,921 Ninth Ave,NULL,Seattle,WA,92101,5555550203,2017-01-01,2017-06-16 +C8C22E25-650B-4FED-8C60-F1D97A7C8FE8,Donovan,Hylton,619 Seventh Ave,NULL,New York,NY,12345,5555550764,2017-01-01,2017-06-21 +75140652-544F-4CE1-8F98-F220BBA8371A,Peter,Marton,639 Palm St,NULL,Portland,OR,85101,5555550661,2017-01-21,2017-06-09 +FE9DC508-6D22-49FD-99DC-F24F1C6D6089,Sofia,Padilla,983 Dogwood St,NULL,New York,NY,12345,5555550587,2017-01-01,2017-09-22 +15BAAA65-8D60-4596-BDBB-F30BEBCB748A,Victoria,Costache,143 Palm St,NULL,Washington,DC,21001,5555559010,2017-01-01,2017-08-31 +310887B6-5FF7-47CE-84A9-F350F9D92B9E,Amelia,Clark,33 Tenth Ave,NULL,Portland,OR,85101,5555559704,2017-01-01,2017-06-04 +396E2801-AB13-4ADF-BB09-F6A53383F63F,Karla,Lowery,360 Pine St,NULL,Austin,TX,75101,5555559586,2017-01-01,2017-07-19 +2445F693-684A-4B5B-BDAD-F6DBCC9EB6B2,Sara,Zogovic,833 Palm St,NULL,Seattle,WA,92101,5555559796,2017-04-15,2017-07-25 +CEBCBA51-F4CC-4852-8B9D-F87255854D7C,Dean,Peric,403 Fourth Ave,NULL,Austin,TX,75101,5555550681,2017-01-01,2017-09-27 +06AA4131-D334-4BA5-BE61-F8F5B5C397F7,Nicholas,Lambert,253 Second Ave,NULL,Portland,OR,85101,5555559183,2017-01-01,2017-10-20 +92EE456C-FDBD-429E-A304-F9C9F5948D12,Millard,Rodman,847 Seventh Ave,NULL,New York,NY,12345,5555550584,2017-02-11,2017-08-06 +E91E882C-CDDF-43FE-845C-FA24D976E7F4,Leonardo,Jozic,322 Second Ave,NULL,Minneapolis,MN,58101,5555559252,2017-01-01,2017-11-11 +224A6AEC-C41A-47C4-8499-FAC57321A82A,Morta,Paugaite,247 Sixth Ave,NULL,Minneapolis,MN,58101,5555550218,2017-05-08,2017-08-18 +70E9F4BF-CF0D-43A6-9337-FCA7AC292ACF,Laima,Gustyte,573 Palm St,NULL,Portland,OR,85101,5555559888,2017-01-01,2017-07-25 +8E6DDDD5-B80B-4A79-8FF4-FD09C0977EA4,Diana,Pace,208 Seventh Ave,NULL,Austin,TX,75101,5555550973,2017-01-01,2017-06-23 +88A05466-55F8-4D4D-9C90-FDEEBE6597D6,Janis,Ferrell,975 Third Ave,NULL,Seattle,WA,92101,5555559735,2017-01-01,2017-10-22 +743C1A10-817C-4495-A5CD-FE41F28C0CEE,Alexandra,Kidd,725 Tenth Ave,NULL,Minneapolis,MN,58101,5555559251,2017-01-01,2017-09-21 \ No newline at end of file diff --git a/examples/ssis/example_ssis_project/PurviewSSISTest.ispac b/examples/ssis/example_ssis_project/PurviewSSISTest.ispac new file mode 100644 index 0000000..55d121d Binary files /dev/null and b/examples/ssis/example_ssis_project/PurviewSSISTest.ispac differ diff --git a/examples/ssis/meta_model/adf/adf_executeSsisPackage_activity.json b/examples/ssis/meta_model/adf/adf_executeSsisPackage_activity.json new file mode 100644 index 0000000..83a0fcd --- /dev/null +++ b/examples/ssis/meta_model/adf/adf_executeSsisPackage_activity.json @@ -0,0 +1,122 @@ +{ + "category": "ENTITY", + "guid": "20788b46-ef1b-bdf1-6745-2b7d553cb2a1", + "createdBy": "admin", + "updatedBy": "admin", + "createTime": 1615787946026, + "updateTime": 1615787946026, + "version": 1, + "name": "adf_executeSsisPackage_activity", + "description": "adf_executeSsisPackage_activity", + "typeVersion": "1.0", + "serviceType": "Azure Data Factory", + "lastModifiedTS": "1", + "attributeDefs": [], + "superTypes": [ + "adf_activity" + ], + "subTypes": [], + "relationshipAttributeDefs": [ + { + "name": "outputs", + "typeName": "array", + "isOptional": true, + "cardinality": "SET", + "valuesMinCount": 0, + "valuesMaxCount": 2147483647, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "relationshipTypeName": "process_dataset_outputs", + "isLegacyAttribute": true + }, + { + "name": "parent", + "typeName": "adf_process", + "isOptional": false, + "cardinality": "SINGLE", + "valuesMinCount": -1, + "valuesMaxCount": -1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "relationshipTypeName": "process_parent", + "isLegacyAttribute": false + }, + { + "name": "inputs", + "typeName": "array", + "isOptional": true, + "cardinality": "SET", + "valuesMinCount": 0, + "valuesMaxCount": 2147483647, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "relationshipTypeName": "dataset_process_inputs", + "isLegacyAttribute": true + }, + { + "name": "associatedPackage", + "typeName": "ssis_package", + "isOptional": true, + "cardinality": "SINGLE", + "valuesMinCount": -1, + "valuesMaxCount": -1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "relationshipTypeName": "adf_executeSsisPackage_activity_associated_package", + "isLegacyAttribute": false + }, + { + "name": "subProcesses", + "typeName": "array", + "isOptional": true, + "cardinality": "SET", + "valuesMinCount": -1, + "valuesMaxCount": -1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "constraints": [ + { + "type": "ownedRef" + } + ], + "relationshipTypeName": "process_parent", + "isLegacyAttribute": false + }, + { + "name": "runInstances", + "typeName": "array", + "isOptional": true, + "cardinality": "SET", + "valuesMinCount": -1, + "valuesMaxCount": -1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "constraints": [ + { + "type": "ownedRef" + } + ], + "relationshipTypeName": "adf_activity_run_instances", + "isLegacyAttribute": false + }, + { + "name": "meanings", + "typeName": "array", + "isOptional": true, + "cardinality": "SET", + "valuesMinCount": -1, + "valuesMaxCount": -1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "relationshipTypeName": "AtlasGlossarySemanticAssignment", + "isLegacyAttribute": false + } + ] +} \ No newline at end of file diff --git a/examples/ssis/meta_model/adf/adf_executeSsisPackage_activity_run.json b/examples/ssis/meta_model/adf/adf_executeSsisPackage_activity_run.json new file mode 100644 index 0000000..d62b8fb --- /dev/null +++ b/examples/ssis/meta_model/adf/adf_executeSsisPackage_activity_run.json @@ -0,0 +1,105 @@ +{ + "category": "ENTITY", + "guid": "618e0397-61ba-801f-bf3a-12d61bde6208", + "createdBy": "admin", + "updatedBy": "admin", + "createTime": 1615787946026, + "updateTime": 1615787946026, + "version": 1, + "name": "adf_executeSsisPackage_activity_run", + "description": "adf_executeSsisPackage_activity_run", + "typeVersion": "1.0", + "serviceType": "Azure Data Factory", + "lastModifiedTS": "1", + "attributeDefs": [], + "superTypes": [ + "adf_activity_run" + ], + "subTypes": [], + "relationshipAttributeDefs": [ + { + "name": "outputs", + "typeName": "array", + "isOptional": true, + "cardinality": "SET", + "valuesMinCount": 0, + "valuesMaxCount": 2147483647, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "relationshipTypeName": "process_dataset_outputs", + "isLegacyAttribute": true + }, + { + "name": "parent", + "typeName": "adf_process", + "isOptional": false, + "cardinality": "SINGLE", + "valuesMinCount": -1, + "valuesMaxCount": -1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "relationshipTypeName": "process_parent", + "isLegacyAttribute": false + }, + { + "name": "activity", + "typeName": "adf_activity", + "isOptional": false, + "cardinality": "SINGLE", + "valuesMinCount": -1, + "valuesMaxCount": -1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "relationshipTypeName": "adf_activity_run_instances", + "isLegacyAttribute": false + }, + { + "name": "inputs", + "typeName": "array", + "isOptional": true, + "cardinality": "SET", + "valuesMinCount": 0, + "valuesMaxCount": 2147483647, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "relationshipTypeName": "dataset_process_inputs", + "isLegacyAttribute": true + }, + { + "name": "subProcesses", + "typeName": "array", + "isOptional": true, + "cardinality": "SET", + "valuesMinCount": -1, + "valuesMaxCount": -1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "constraints": [ + { + "type": "ownedRef" + } + ], + "relationshipTypeName": "process_parent", + "isLegacyAttribute": false + }, + { + "name": "meanings", + "typeName": "array", + "isOptional": true, + "cardinality": "SET", + "valuesMinCount": -1, + "valuesMaxCount": -1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "relationshipTypeName": "AtlasGlossarySemanticAssignment", + "isLegacyAttribute": false + } + ] +} +Lea \ No newline at end of file diff --git a/examples/ssis/meta_model/adf/adf_executeSsisPackage_operation.json b/examples/ssis/meta_model/adf/adf_executeSsisPackage_operation.json new file mode 100644 index 0000000..29ac3a1 --- /dev/null +++ b/examples/ssis/meta_model/adf/adf_executeSsisPackage_operation.json @@ -0,0 +1,91 @@ +{ + "category": "ENTITY", + "guid": "39354631-a72e-34d4-2ff2-af6806a52ccd", + "createdBy": "admin", + "updatedBy": "admin", + "createTime": 1615787946026, + "updateTime": 1615787946026, + "version": 1, + "name": "adf_executeSsisPackage_operation", + "description": "adf_executeSsisPackage_operation", + "typeVersion": "1.0", + "serviceType": "Azure Data Factory", + "lastModifiedTS": "1", + "attributeDefs": [], + "superTypes": [ + "adf_activity_operation" + ], + "subTypes": [], + "relationshipAttributeDefs": [ + { + "name": "outputs", + "typeName": "array", + "isOptional": true, + "cardinality": "SET", + "valuesMinCount": 0, + "valuesMaxCount": 2147483647, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "relationshipTypeName": "process_dataset_outputs", + "isLegacyAttribute": true + }, + { + "name": "parent", + "typeName": "adf_process", + "isOptional": false, + "cardinality": "SINGLE", + "valuesMinCount": -1, + "valuesMaxCount": -1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "relationshipTypeName": "process_parent", + "isLegacyAttribute": false + }, + { + "name": "inputs", + "typeName": "array", + "isOptional": true, + "cardinality": "SET", + "valuesMinCount": 0, + "valuesMaxCount": 2147483647, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "relationshipTypeName": "dataset_process_inputs", + "isLegacyAttribute": true + }, + { + "name": "subProcesses", + "typeName": "array", + "isOptional": true, + "cardinality": "SET", + "valuesMinCount": -1, + "valuesMaxCount": -1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "constraints": [ + { + "type": "ownedRef" + } + ], + "relationshipTypeName": "process_parent", + "isLegacyAttribute": false + }, + { + "name": "meanings", + "typeName": "array", + "isOptional": true, + "cardinality": "SET", + "valuesMinCount": -1, + "valuesMaxCount": -1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "relationshipTypeName": "AtlasGlossarySemanticAssignment", + "isLegacyAttribute": false + } + ] +} \ No newline at end of file diff --git a/examples/ssis/meta_model/adf/ssis_package.json b/examples/ssis/meta_model/adf/ssis_package.json new file mode 100644 index 0000000..710e76c --- /dev/null +++ b/examples/ssis/meta_model/adf/ssis_package.json @@ -0,0 +1,81 @@ +{ + "category": "ENTITY", + "guid": "fd27d941-92e0-de1c-04b4-a9e59d57e68c", + "createdBy": "admin", + "updatedBy": "admin", + "createTime": 1615787948725, + "updateTime": 1615787948725, + "version": 1, + "name": "ssis_package", + "description": "ssis_package", + "typeVersion": "1.0", + "serviceType": "SQL Server Integration Services", + "options": { + "derivedLineageSources": "[\"subProcesses\"]" + }, + "lastModifiedTS": "1", + "attributeDefs": [ + { + "name": "lastRunTime", + "typeName": "date", + "isOptional": true, + "cardinality": "SINGLE", + "valuesMinCount": 0, + "valuesMaxCount": 1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false + } + ], + "superTypes": [ + "Asset" + ], + "subTypes": [], + "relationshipAttributeDefs": [ + { + "name": "associatedActivities", + "typeName": "array", + "isOptional": true, + "cardinality": "SET", + "valuesMinCount": -1, + "valuesMaxCount": -1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "relationshipTypeName": "adf_executeSsisPackage_activity_associated_package", + "isLegacyAttribute": false + }, + { + "name": "subProcesses", + "typeName": "array", + "isOptional": true, + "cardinality": "SET", + "valuesMinCount": -1, + "valuesMaxCount": -1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "constraints": [ + { + "type": "ownedRef" + } + ], + "relationshipTypeName": "ssis_package_process_parent", + "isLegacyAttribute": false + }, + { + "name": "meanings", + "typeName": "array", + "isOptional": true, + "cardinality": "SET", + "valuesMinCount": -1, + "valuesMaxCount": -1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "relationshipTypeName": "AtlasGlossarySemanticAssignment", + "isLegacyAttribute": false + } + ] +} +L \ No newline at end of file diff --git a/examples/ssis/meta_model/adf/ssis_package_process.json b/examples/ssis/meta_model/adf/ssis_package_process.json new file mode 100644 index 0000000..95c0d4b --- /dev/null +++ b/examples/ssis/meta_model/adf/ssis_package_process.json @@ -0,0 +1,88 @@ +{ + "category": "ENTITY", + "guid": "86316cc5-78b6-dea4-dbf7-2d0f3d3589b9", + "createdBy": "admin", + "updatedBy": "admin", + "createTime": 1615787948725, + "updateTime": 1615787948725, + "version": 1, + "name": "ssis_package_process", + "description": "ssis_package_process", + "typeVersion": "1.0", + "serviceType": "SQL Server Integration Services", + "options": { + "defaultRenderedLineage": "parent" + }, + "lastModifiedTS": "1", + "attributeDefs": [ + { + "name": "columnMapping", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "valuesMinCount": 0, + "valuesMaxCount": 1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false + } + ], + "superTypes": [ + "Process" + ], + "subTypes": [], + "relationshipAttributeDefs": [ + { + "name": "outputs", + "typeName": "array", + "isOptional": true, + "cardinality": "SET", + "valuesMinCount": 0, + "valuesMaxCount": 2147483647, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "relationshipTypeName": "process_dataset_outputs", + "isLegacyAttribute": true + }, + { + "name": "parent", + "typeName": "ssis_package", + "isOptional": false, + "cardinality": "SINGLE", + "valuesMinCount": -1, + "valuesMaxCount": -1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "relationshipTypeName": "ssis_package_process_parent", + "isLegacyAttribute": false + }, + { + "name": "inputs", + "typeName": "array", + "isOptional": true, + "cardinality": "SET", + "valuesMinCount": 0, + "valuesMaxCount": 2147483647, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "relationshipTypeName": "dataset_process_inputs", + "isLegacyAttribute": true + }, + { + "name": "meanings", + "typeName": "array", + "isOptional": true, + "cardinality": "SET", + "valuesMinCount": -1, + "valuesMaxCount": -1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "relationshipTypeName": "AtlasGlossarySemanticAssignment", + "isLegacyAttribute": false + } + ] +} diff --git a/examples/ssis/meta_model/example_templates/legacy_ssis_package_process_template.json b/examples/ssis/meta_model/example_templates/legacy_ssis_package_process_template.json new file mode 100644 index 0000000..054de17 --- /dev/null +++ b/examples/ssis/meta_model/example_templates/legacy_ssis_package_process_template.json @@ -0,0 +1,47 @@ +{ + "typeName": "legacy_ssis_package_process", + "attributes": { + "outputs": [ + { + "uniqueAttributes": { + "qualifiedName": "" + } + } + ], + "inputs": [ + { + "uniqueAttributes": { + "qualifiedName": "" + } + } + ], + "qualifiedName": "", + "name": "SSIS_Movie_Loader", + "columnMapping": "" + }, + "status": "ACTIVE", + "relationshipAttributes": { + "outputs": [ + { + "qualifiedName": "", + "entityStatus": "ACTIVE", + "displayText": "" + } + ], + "parent": { + "typeName": "legacy_ssis_package", + "displayText": "SSIS Movie Loader Package", + "relationshipType": "legacy_ssis_package_process_parent", + "relationshipAttributes": { + "typeName": "legacy_ssis_package_process_parent" + } + }, + "inputs": [ + { + "qualifiedName": "", + "entityStatus": "ACTIVE", + "displayText": "" + } + ] + } +} \ No newline at end of file diff --git a/examples/ssis/meta_model/example_templates/legacy_ssis_package_template.json b/examples/ssis/meta_model/example_templates/legacy_ssis_package_template.json new file mode 100644 index 0000000..9c5c9d4 --- /dev/null +++ b/examples/ssis/meta_model/example_templates/legacy_ssis_package_template.json @@ -0,0 +1,9 @@ +{ + "typeName": "legacy_ssis_package_new_copy", + "attributes": { + "qualifiedName": "", + "name": "", + "description": "", + "lastRunTime": "" + } +} \ No newline at end of file diff --git a/examples/ssis/meta_model/example_types/legacy_ssis_package.json b/examples/ssis/meta_model/example_types/legacy_ssis_package.json new file mode 100644 index 0000000..3d30f27 --- /dev/null +++ b/examples/ssis/meta_model/example_types/legacy_ssis_package.json @@ -0,0 +1,67 @@ +{ + "category": "ENTITY", + "guid": "a8cc9ce2-ccab-4836-8622-74224d9df934", + "createdBy": "b463e44d-4c0e-4cfd-982b-0d3de4c5197e", + "updatedBy": "b463e44d-4c0e-4cfd-982b-0d3de4c5197e", + "createTime": 1620398542506, + "updateTime": 1620410414735, + "version": 4, + "name": "legacy_ssis_package_new", + "description": "legacy_ssis_package_new", + "typeVersion": "1.0", + "serviceType": "Legacy SQL Server Integration Services", + "options": { + "derivedLineageSources": "[\"subProcesses\"]" + }, + "lastModifiedTS": "4", + "attributeDefs": [ + { + "name": "lastRunTime", + "typeName": "date", + "isOptional": true, + "cardinality": "SINGLE", + "valuesMinCount": 0, + "valuesMaxCount": 1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false + } + ], + "superTypes": [ + "Asset" + ], + "subTypes": [], + "relationshipAttributeDefs": [ + { + "name": "subProcesses", + "typeName": "array", + "isOptional": true, + "cardinality": "SET", + "valuesMinCount": -1, + "valuesMaxCount": -1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "constraints": [ + { + "type": "ownedRef" + } + ], + "relationshipTypeName": "legacy_ssis_package_process_new_parent", + "isLegacyAttribute": false + }, + { + "name": "meanings", + "typeName": "array", + "isOptional": true, + "cardinality": "SET", + "valuesMinCount": -1, + "valuesMaxCount": -1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "relationshipTypeName": "AtlasGlossarySemanticAssignment", + "isLegacyAttribute": false + } + ] +} \ No newline at end of file diff --git a/examples/ssis/meta_model/example_types/legacy_ssis_package_process.json b/examples/ssis/meta_model/example_types/legacy_ssis_package_process.json new file mode 100644 index 0000000..8c3130f --- /dev/null +++ b/examples/ssis/meta_model/example_types/legacy_ssis_package_process.json @@ -0,0 +1,88 @@ +{ + "category": "ENTITY", + "guid": "be732dde-8ecf-45da-b5a1-afe739d10917", + "createdBy": "b463e44d-4c0e-4cfd-982b-0d3de4c5197e", + "updatedBy": "b463e44d-4c0e-4cfd-982b-0d3de4c5197e", + "createTime": 1620398542498, + "updateTime": 1620410414688, + "version": 4, + "name": "legacy_ssis_package_process_new", + "description": "legacy_ssis_package_process_new", + "typeVersion": "1.0", + "serviceType": "Legacy SQL Server Integration Services", + "options": { + "defaultRenderedLineage": "parent" + }, + "lastModifiedTS": "4", + "attributeDefs": [ + { + "name": "columnMapping", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "valuesMinCount": 0, + "valuesMaxCount": 1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false + } + ], + "superTypes": [ + "Process" + ], + "subTypes": [], + "relationshipAttributeDefs": [ + { + "name": "outputs", + "typeName": "array", + "isOptional": true, + "cardinality": "SET", + "valuesMinCount": 0, + "valuesMaxCount": 2147483647, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "relationshipTypeName": "process_dataset_outputs", + "isLegacyAttribute": true + }, + { + "name": "parent", + "typeName": "legacy_ssis_package_new", + "isOptional": false, + "cardinality": "SINGLE", + "valuesMinCount": -1, + "valuesMaxCount": -1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "relationshipTypeName": "legacy_ssis_package_process_new_parent", + "isLegacyAttribute": false + }, + { + "name": "inputs", + "typeName": "array", + "isOptional": true, + "cardinality": "SET", + "valuesMinCount": 0, + "valuesMaxCount": 2147483647, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "relationshipTypeName": "dataset_process_inputs", + "isLegacyAttribute": true + }, + { + "name": "meanings", + "typeName": "array", + "isOptional": true, + "cardinality": "SET", + "valuesMinCount": -1, + "valuesMaxCount": -1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "relationshipTypeName": "AtlasGlossarySemanticAssignment", + "isLegacyAttribute": false + } + ] +} \ No newline at end of file diff --git a/examples/ssis/meta_model/example_types/legacy_ssis_package_process_parent.json b/examples/ssis/meta_model/example_types/legacy_ssis_package_process_parent.json new file mode 100644 index 0000000..ef3ae37 --- /dev/null +++ b/examples/ssis/meta_model/example_types/legacy_ssis_package_process_parent.json @@ -0,0 +1,31 @@ +{ + "category": "RELATIONSHIP", + "guid": "bfbec236-da6f-d0be-7add-5ba42271bdd8", + "createdBy": "b463e44d-4c0e-4cfd-982b-0d3de4c5197e", + "updatedBy": "b463e44d-4c0e-4cfd-982b-0d3de4c5197e", + "createTime": 1625682387019, + "updateTime": 1625682387019, + "version": 1, + "name": "legacy_ssis_package_process_parent_copy", + "description": "legacy_ssis_package_process_parent_copy", + "typeVersion": "1.0", + "serviceType": "Legacy SQL Server Integration Services", + "lastModifiedTS": "1", + "attributeDefs": [], + "relationshipCategory": "COMPOSITION", + "propagateTags": "NONE", + "endDef1": { + "type": "legacy_ssis_package_process_copy", + "name": "parent", + "isContainer": false, + "cardinality": "SINGLE", + "isLegacyAttribute": false + }, + "endDef2": { + "type": "legacy_ssis_package", + "name": "subProcesses", + "isContainer": true, + "cardinality": "SET", + "isLegacyAttribute": false + } +} \ No newline at end of file diff --git a/examples/ssis/scripts/powershell_loop_script.ps1 b/examples/ssis/scripts/powershell_loop_script.ps1 new file mode 100644 index 0000000..64ea0c3 --- /dev/null +++ b/examples/ssis/scripts/powershell_loop_script.ps1 @@ -0,0 +1,156 @@ +Param( + [Parameter (Mandatory = $false)] + [object] $WebHookData +) +if($WebHookData){ + $body = ConvertFrom-Json -InputObject $WebHookData.RequestBody + $StorageAccountName = ($body | Select -ExpandProperty "StorageAccountName") + $FolderName = ($body | Select -ExpandProperty "FolderName") + $ProjectName = ($body | Select -ExpandProperty "ProjectName") + $FileSystemName = ($body | Select -ExpandProperty "FileSystemName") + $dirname = ($body | Select -ExpandProperty "dirname") + $VaultName = ($body | Select -ExpandProperty "VaultName") + $ADLSecretName = ($body | Select -ExpandProperty "ADLSecretName") + $SQLSecretName = ($body | Select -ExpandProperty "SQLSecretName") +} + +# Import-Module SqlServer +# Import-Module -Name Az.Storage +$assemblylist = +"Microsoft.SqlServer.Management.Common", +"Microsoft.SqlServer.Smo", +"Microsoft.SqlServer.Dmf ", +"Microsoft.SqlServer.Instapi ", +"Microsoft.SqlServer.SqlWmiManagement ", +"Microsoft.SqlServer.ConnectionInfo ", +"Microsoft.SqlServer.SmoExtended ", +"Microsoft.SqlServer.SqlTDiagM ", +"Microsoft.SqlServer.SString ", +"Microsoft.SqlServer.Management.RegisteredServers ", +"Microsoft.SqlServer.Management.Sdk.Sfc ", +"Microsoft.SqlServer.SqlEnum ", +"Microsoft.SqlServer.RegSvrEnum ", +"Microsoft.SqlServer.WmiEnum ", +"Microsoft.SqlServer.ServiceBrokerEnum ", +"Microsoft.SqlServer.ConnectionInfoExtended ", +"Microsoft.SqlServer.Management.Collector ", +"Microsoft.SqlServer.Management.CollectorEnum", +"Microsoft.SqlServer.Management.Dac", +"Microsoft.SqlServer.Management.DacEnum", +"Microsoft.SqlServer.Management.Utility" + +foreach ($asm in $assemblylist) +{ +$asm = [Reflection.Assembly]::LoadWithPartialName($asm) +} + +$server = New-Object Microsoft.SqlServer.Management.Smo.Server; + + +try{ +$server.ConnectionContext.ConnectionString = $SQLSecretName + + + + +$server.ConnectionContext.Connect() +"worked" +}catch +{"didnt work "} + +$ctx = New-AzStorageContext -StorageAccountName $StorageAccountName -StorageAccountKey $ADLSecretName + +$instanceName = "SERVER\SQL"; +$bufferSize = 8192; + +$File = New-TemporaryFile +Remove-Item -path $File -force + + +$baseExportSql = "EXEC [SSISDB].[catalog].[get_project] @folder_name = $FolderName, @Project_name = $ProjectName;"; + +$connection = New-Object System.Data.SqlClient.SqlConnection $SQLSecretName; +$command = New-Object System.Data.SqlClient.SqlCommand; +$command.Connection = $connection; +Write-Host $connection + +$connection.Folders | % { + + $folder = $_; + + $_.Projects | % { + + $project = $_; + + $project.Packages | % { + + "$($folder.Name)\$($project.Name)\$($_.Name)"; + + } + + # create a folder for it + $path = "$($ENV:Temp)\$($File.Name)\$($instanceName)\$($folder.Name)\$($project.Name)"; + New-Item -ItemType Directory -Force -Path $path | Out-Null; + $fileName = "$($path)\$($project.Name).zip"; + Write-Host $fileName + if ($connection.State -ne "Open") + { + $connection.Open(); + } + + $fileOutput = [array]::CreateInstance('Byte', $bufferSize) + + $command.CommandText = $baseExportSql -f $folder.Name, $project.Name; + $reader = $command.ExecuteReader(); + + while($reader.Read()) + { + $fileStream = New-Object System.IO.FileStream($fileName), Create, Write; + $binaryWriter = New-Object System.IO.BinaryWriter $fileStream; + + $start = 0; + + $received = $reader.GetBytes(0, $start, $fileOutput, 0, $bufferSize - 1); + + while ($received -gt 0) + { + $binaryWriter.Write($fileOutput, 0, $received); + $binaryWriter.Flush(); + $start += $received; + + $received = $reader.GetBytes(0, $start, $fileOutput, 0, $bufferSize - 1); + } + + $binaryWriter.Close(); + $fileStream.Close(); + } + + $reader.Close(); + } + +} + +if ($connection.State -eq "Open") +{ + $connection.Close(); +} + + +Expand-Archive -LiteralPath $fileName -DestinationPath "$($ENV:Temp)\UnZipSSIS\folder" + +$folderContent = Get-ChildItem -Path "$($ENV:Temp)\UnZipSSIS\folder\" -Name + +foreach ($file in $folderContent){ + $destPath = $dirname + (Get-Item "$($ENV:Temp)\UnZipSSIS\folder\$file").Name + New-AzDataLakeGen2Item -Context $ctx -FileSystem $FileSystemName -Path $destPath -Source "$($ENV:Temp)\UnZipSSIS\folder\$file" -Force +} + +Remove-Item "$($ENV:Temp)\UnZipSSIS" -Recurse -Force -Confirm:$false +Remove-Item "$($ENV:Temp)\$($File.Name)" -Recurse -Force -Confirm:$false + +if($WebHookData){ + $body = ConvertFrom-Json -InputObject $WebHookData.RequestBody + $Uri = ($body | Select -ExpandProperty "callBackUri") + print($Uri) + Invoke-WebRequest -Uri $Uri -Method Post +} \ No newline at end of file diff --git a/examples/ssis/ssis.md b/examples/ssis/ssis.md new file mode 100644 index 0000000..9d61f3b --- /dev/null +++ b/examples/ssis/ssis.md @@ -0,0 +1,131 @@ +# Overview of the SSIS example + +As described in the [Readme.md](../../README.md#connector-development-process) document, the development steps for creating any connector can be divided into the following categories: + +1. Locate and pull meta-data +2. Define the meta-model +3. Parse / transform the meta-data +4. Ingest the meta-data into Purview + +## Pull SSIS Meta-data + +The first step in creating a custom connector is locating the target source metadata, and methods for retrieving this data programatically. SSIS metadata is primarily contained within a project ispac file which is exportable from the SQL Server on which SSIS in installed. This file is actually a zip based archive and contains the following files: + +* **@Project_manifest** + * This file includes information about the authoring of the package as well as parameters, server info, etc. +* **Connection manager files (conmgr extension)** + * These files detail connections defined for the package, containing connection types, paths, etc. Server names and connetion information is overwritten by package configuration parameters in our scenario. +* **Package files (dtsx extension)** + * XML file containing all of the details about the SSIS job. The large majority of the meta-data is contained in this file. + +Other files are not used for meta-data extraction in this example, but files like Project.params would be useful in situations where parameters were assigned at the project level. +In our scenario, we are using package level parameters which are defined by configuring the dtsx package. These parameters are dynamic and not available as part of the exported package data. Values for these parameters must be retrieved from the SSISDB database. The SQL query and code for this retrieval is contained in the SSISDB_Get_Params notebook. + +In order to export the SSIS package programmatically, the exporting code needs to authenticate to the SQL Server using Windows Authentication. This causes issues when trying to authenticate outside of the domain / network of the VM. In order to log on locally, the accelerator uses Azure Automation combined with Log Analytics to create a [hybrid runbook](https://docs.microsoft.com/en-us/azure/automation/automation-hybrid-runbook-worker) which will run locally on the VM, using the log analytics agent. This hybrid runbook is analogous to the self-hosted integration runtime (SHIR) component which retrieves meta-data for supported, on-premises sources. + +## Define the SSIS Meta-model + +Developing a meta-model for SSIS is helped by the fact that Purview does support SSIS when implemented through Azure Data Factory. This means that there are already existing types built into Purview which can be cloned to create the on-prem SSIS meta-model. Instructions for cloning these types using the [Purview Custom Types Tool](https://github.com/microsoft/Purview-Custom-Types-Tool-Solution-Accelerator) are included as part of the "Running the Solution" section below. Examples from the ADF implementation are included in the ./meta-model/adf directory. + +## Parsing the SSIS Package + +The main source of meta-data for the SSIS project is the SSIS package file (dtsx). This xml file contains all the information about the data transfer, column names, etc. To make it easier to parse this file, it is translated into json format as part of the pipeline processing. From there, meta-data is extracted as needed to fill in the Purview entity templates which are modifications of those created by the Purview Custom Types tool. These templates can be viewed in the ssis/meta_model/example_templates directory. +One of the critical translations is from the name of the SSIS source or target to a qualified name for that source or target within Purview. Purview treats these qualified names as unique and it is very possible that the entities contained in the SSIS package already have representation in Purview if those sources have been scanned. For example, in our simple package we are using an ADLS csv file as a source. if that ADLS storage has already been scanned by Purview, it is necessary to make sure that the SSIS source csv is recognized as the same source entity as the ADLS csv which Purview has scanned. This requires carful creation of Purview qualified names from the SSIS source target names in the package. +Support has been built into the Purview Connector Services to create a dummy entity to represent sources or targets of SSIS packages that do not exist in Puview, and to delete and map these to sources that do exist if they are scanned into Purview at a later time. For details, see the [Purview Connector Services](../../purview_connector_services/purview_connector_services.md) documentation. + +## Overview of the ingestion / parsing pipeline + +![ssis_pipeline.png](../../assets/images/ssis_pipeline.png) + +1. Retrieve the SQL Connection String, Storage Account Key, and SQL Password from KeyVault, and read them into the pipeline variables. +2. Run the SISSDB_Get_Params notebook which does the following + 1. Queries the SSISDB database to get the last execution ID + 2. Compares this value against a stored execution ID value. If the package has not been executed since the last scan, returns false and exits the notebook + 3. If the SSIS package has been run since last scan, pulls the parameters for the package from the SSISDB and copies them to storage + 4. Returns true to continue running the rest of the pipeline +3. Run the PowerShell_WebHook task which triggers a Hybrid Runbook (defined in the Azure Automation Service) to be run locally on the SQL Server VM. The PowerShell Script is deployed from ssis/scripts/powershell_loop_script.ps1. This script: + 1. Exports the target SSIS package(s) + 2. Extracts the package and copies the resulting files into blob storage under pccsa_main/ssis-connector/ssis-package +4. Transform the XML SSIS package and connectors information to JSON to make it easier to work with programmatically in Python +5. Run the SSIS_Scan_Package notebook to: + 1. parse the JSON files and transform them into the Purview types + 2. Copy the resulting entity data into the pccsa_main/incoming directory in blob storage + +## Running the Example Solution + +### Follow the instructions for deploying the prerequisites, base service, and SSIS example + +### Configure and run the SSIS package on the VM + +* RDP into the SQL VM and start SQL Server Management Studio +* In the Object Explorer, navigate to the PurviewSSISTest project, right click on it and choose 'Configure...' + ![smss_config_project.png](../../assets/images/smss_config_project.png) +* In configuration properties, select the 'Connection Managers' tab +* Configure the following to match your installation for the ADLS Connection + * AccountKey - Azure Storage Account Key + * AccountName - Azure Storage Account Name + * ConnectionString - IMPORTANT: Copy the existing string and modify only the account name + ![smss_adls_config.png](../../assets/images/smss_adls_config.png) +* Switch to the SQL OLEDB Configuration and Configure the following to match your installation + * Password - SQL password (from deployment script settings file) + * ServerName - SQL VM DNS name or IP + * ConnectionString - IMPORTANT: Copy the existing string and modify only the SQL name or IP + ![smss_oledb_config.png](../../assets/images/smss_oledb_config.png) +* Right click the project again and choose 'Validate...' +* Navigate to Package.dtsx under the PurviewSSISTest project, right click it and choose 'Execute...' +* Choose 'Yes' to view the report and ensure that the job ran successfully. You should now have data in the purview-sqlmi-db/dbo.MovCustomers table + +### Create types in Purview + +For general Atlas type information review the resource list in [README.MD](../../README.MD#purview-development-resources) + +* After configuring and starting the Purview Custom Types tool, you will be presented with a drop down allowing you to create a new service type (think of this like a grouping of all the types for a particular connector project), or a view of all types. Since we will be cloning existing SSIS types, choose 'All Type Definitions' + ![pcttsa_select_service_type.png](../../assets/images/pcttsa_select_service_type.png) +* Choose 'Entity Defs' on the left pane to see all of the main entity definition types. Press Ctrl+'F' to find types containing 'SSIS' in the type name. Locate and select the ssis_package type + + ![pcttsa_clone_type_btn.png](../../assets/images/pcttsa_clone_type_btn.png) + +* Choose the option to clone the type + + ![pcttsa_clone_type.png](../../assets/images/pcttsa_clone_type.png) + +* In the 'Service Type' dropdown, choose the option to 'Create New Service Type' and name the new service type legacy_ssis +Rename the type from ssis_package_copy to legacy_ssis_package and choose the option to save it to Azure (saves the type to Purview) + + ![pcttsa_create_new_service_type.png](../../assets/images/pcttsa_create_new_service_type.png) + +* Repeat these steps to clone the ssis_package_process type to legacy_ssis_package_process, also under the legacy_ssis service type +* In the left pane, collapse 'Entity Def' list and expand the 'Relationship Defs' list + + ![pcttsa_relationship_defs.png](../../assets/images/pcttsa_relationship_defs.png) + +* Use the find functionality to search again on 'ssis' and locate the ssis_package_process_parent relationship type. This relationship type defines the relationship between the ssis types, and defines the inputs and outputs necessary for lineage creation. Clone and rename this type to purview also under the legacy_ssis service type. +* Select the legacy_ssis_package_process type in the dropdown next to EndDef1 +* Select the legacy_ssis_package type in the dropdown next to EndDef2 +* Save the type to Purview + + ![pcttsa_relationship.png](../../assets/images/pcttsa_relationship.png) + +* In the upper left of the interface, select 'Home' - you will be presented again with the initial drop down +* This time choose to view the 'legacy_ssis' service types +* For each of the two entity types you just created, select them and choose the "create template" option + + ![pcttsa_create_template.png](../../assets/images/pcttsa_create_template.png) + +* The Custom Type Tool will create templates in blob storage under 'custom-typedefs'. These templates are modified to form the final templates used for transformation to Purview types in the pipeline + +### Run the SSIS Pipeline + +* In Synapse Analytics Workspace, navigate to the SSIS_Package_Pipeline. All parameters and variables should be preconfigured. You can click on the pipeline background and choose the Parameters tab to examine the values + + ![run_ssis_pipeline.png](../../assets/images/run_ssis_pipeline.png) + +* Start SSIS pipeline by triggering it, or pressing the debug button. Note that the pipeline run can easily be triggered on a schedule allowing for periodic updating of any changes to the SSIS activity +* Examine the results for each pipeline activity + +### View Results in Purview + +* Once the pipeline completes, it will copy the entity data to the incoming folder in blob storage, this will activate the Purview Load Custom Types Pipeline to ingest the data into Purview +* Open Purview Studio and search for the newly created lineage for the legacy SSIS job + +![purview_ssis_lineage.svg](../../assets/images/purview_ssis_lineage.png) diff --git a/examples/ssis/synapse/dataset/Json1.json b/examples/ssis/synapse/dataset/Json1.json new file mode 100644 index 0000000..4e31d78 --- /dev/null +++ b/examples/ssis/synapse/dataset/Json1.json @@ -0,0 +1,20 @@ +{ + "name": "Json1", + "properties": { + "linkedServiceName": { + "referenceName": "purviewaccws-WorkspaceDefaultStorage", + "type": "LinkedServiceReference" + }, + "annotations": [], + "type": "Json", + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "folderPath": "pccsa_main/ssis-connector/working", + "fileSystem": "pccsa" + } + }, + "schema": {} + }, + "type": "Microsoft.Synapse/workspaces/datasets" +} \ No newline at end of file diff --git a/examples/ssis/synapse/dataset/Xml1.json b/examples/ssis/synapse/dataset/Xml1.json new file mode 100644 index 0000000..9309611 --- /dev/null +++ b/examples/ssis/synapse/dataset/Xml1.json @@ -0,0 +1,23 @@ +{ + "name": "Xml1", + "properties": { + "linkedServiceName": { + "referenceName": "purviewaccws-WorkspaceDefaultStorage", + "type": "LinkedServiceReference" + }, + "annotations": [], + "type": "Xml", + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "fileName": { + "value": "endswith(dtsx)", + "type": "Expression" + }, + "folderPath": "pccsa_main/ssis-connector/ssis-package", + "fileSystem": "pccsa" + } + } + }, + "type": "Microsoft.Synapse/workspaces/datasets" +} \ No newline at end of file diff --git a/examples/ssis/synapse/dataset/Xml2.json b/examples/ssis/synapse/dataset/Xml2.json new file mode 100644 index 0000000..687ab9a --- /dev/null +++ b/examples/ssis/synapse/dataset/Xml2.json @@ -0,0 +1,23 @@ +{ + "name": "Xml2", + "properties": { + "linkedServiceName": { + "referenceName": "purviewaccws-WorkspaceDefaultStorage", + "type": "LinkedServiceReference" + }, + "annotations": [], + "type": "Xml", + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "fileName": { + "value": "endswith(conmgr)", + "type": "Expression" + }, + "folderPath": "pccsa_main/ssis-connector/ssis-package", + "fileSystem": "pccsa" + } + } + }, + "type": "Microsoft.Synapse/workspaces/datasets" +} \ No newline at end of file diff --git a/examples/ssis/synapse/notebook/SSISDB_Get_Params.ipynb b/examples/ssis/synapse/notebook/SSISDB_Get_Params.ipynb new file mode 100644 index 0000000..3896852 --- /dev/null +++ b/examples/ssis/synapse/notebook/SSISDB_Get_Params.ipynb @@ -0,0 +1,482 @@ +{ + "nbformat": 4, + "nbformat_minor": 2, + "cells": [ + { + "cell_type": "code", + "execution_count": 39, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.livy.statement-meta+json": { + "spark_pool": "notebookrun", + "session_id": 19, + "statement_id": 39, + "state": "finished", + "livy_statement_state": "available", + "queued_time": "2021-08-26T01:02:54.5195026Z", + "session_start_time": null, + "execution_start_time": "2021-08-26T01:02:54.9238761Z", + "execution_finish_time": "2021-08-26T01:02:54.9239769Z" + }, + "text/plain": "StatementMeta(notebookrun, 19, 39, Finished, Available)" + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": "" + } + ], + "metadata": {}, + "source": [ + "import sqlite3\r\n", + "import pyodbc\r\n", + "import csv\r\n", + "import pandas as pd\r\n", + "import datetime\r\n", + "from azure.storage.blob import BlobClient" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.livy.statement-meta+json": { + "spark_pool": "notebookrun", + "session_id": 19, + "statement_id": 40, + "state": "finished", + "livy_statement_state": "available", + "queued_time": "2021-08-26T01:02:54.5924961Z", + "session_start_time": null, + "execution_start_time": "2021-08-26T01:02:55.0128538Z", + "execution_finish_time": "2021-08-26T01:02:55.0129401Z" + }, + "text/plain": "StatementMeta(notebookrun, 19, 40, Finished, Available)" + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": "" + } + ], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true, + "tags": [ + "parameters" + ] + }, + "source": [ + "server = ''\r\n", + "database = ''\r\n", + "username = ''\r\n", + "password = ''\r\n", + "driver = ''\r\n", + "blob_file_path = ''\r\n", + "execution_id_path = ''\r\n", + "blob_account_name = ''\r\n", + "blob_account_key = ''\r\n", + "container = ''\r\n", + "SSISProject = ''" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.livy.statement-meta+json": { + "spark_pool": "notebookrun", + "session_id": 19, + "statement_id": 41, + "state": "finished", + "livy_statement_state": "available", + "queued_time": "2021-08-26T01:02:54.6563104Z", + "session_start_time": null, + "execution_start_time": "2021-08-26T01:02:55.1094239Z", + "execution_finish_time": "2021-08-26T01:02:55.6019139Z" + }, + "text/plain": "StatementMeta(notebookrun, 19, 41, Finished, Available)" + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": "" + } + ], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + "from pyspark.conf import SparkConf\r\n", + "from pyspark.sql import SparkSession\r\n", + "import os\r\n", + "\r\n", + "app_name = 'SSISDB_Get_Params'\r\n", + "my_jars = os.environ.get(\"SPARK_HOME\")\r\n", + "myconf = SparkConf()\r\n", + "myconf.setMaster(\"local\").setAppName(\"SSISDB_Get_Params\")\r\n", + "myconf.set(\"spark.jars\",\"%s/jars/log4j-1.2.17.jar\" % my_jars)\r\n", + "spark = SparkSession\\\r\n", + " .builder\\\r\n", + " .appName(\"SSISDB_Get_Params\")\\\r\n", + " .config(conf = myconf) \\\r\n", + " .getOrCreate()\r\n", + "\r\n", + "\r\n", + "Logger= spark._jvm.org.apache.log4j.Logger\r\n", + "mylogger = Logger.getLogger(app_name)" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.livy.statement-meta+json": { + "spark_pool": "notebookrun", + "session_id": 19, + "statement_id": 42, + "state": "finished", + "livy_statement_state": "available", + "queued_time": "2021-08-26T01:02:54.7154516Z", + "session_start_time": null, + "execution_start_time": "2021-08-26T01:02:55.691253Z", + "execution_finish_time": "2021-08-26T01:02:55.84062Z" + }, + "text/plain": "StatementMeta(notebookrun, 19, 42, Finished, Available)" + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": "" + } + ], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + "def log_msgs(msg_type,msg):\r\n", + " \r\n", + " if msg_type.upper() == \"ERROR\":\r\n", + " print('ERROR: %s' % repr(msg))\r\n", + " mylogger.error(repr(msg))\r\n", + " else:\r\n", + " print('INFO: %s' % repr(msg))\r\n", + " mylogger.info(repr(msg))" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.livy.statement-meta+json": { + "spark_pool": "notebookrun", + "session_id": 19, + "statement_id": 43, + "state": "finished", + "livy_statement_state": "available", + "queued_time": "2021-08-26T01:02:54.793267Z", + "session_start_time": null, + "execution_start_time": "2021-08-26T01:02:55.9294673Z", + "execution_finish_time": "2021-08-26T01:02:56.0898438Z" + }, + "text/plain": "StatementMeta(notebookrun, 19, 43, Finished, Available)" + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": "INFO: 'Entities Created/Updated'\nINFO: ''\nINFO: ''\nINFO: ''\nINFO: ''\nINFO: ''\nINFO: ''\nINFO: ''\nINFO: ''\nINFO: ''\nINFO: ''\nINFO: ''\nINFO: 'DRIVER=;SERVER=;DATABASE=;UID=;PWD='" + } + ], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + "log_msgs(\"INFO\",'Entities Created/Updated')\r\n", + "log_msgs(\"INFO\", server)\r\n", + "log_msgs(\"INFO\",database)\r\n", + "log_msgs(\"INFO\",username)\r\n", + "log_msgs(\"INFO\",password)\r\n", + "log_msgs(\"INFO\",driver)\r\n", + "log_msgs(\"INFO\",blob_file_path)\r\n", + "log_msgs(\"INFO\",execution_id_path)\r\n", + "log_msgs(\"INFO\",blob_account_name)\r\n", + "log_msgs(\"INFO\",blob_account_key)\r\n", + "log_msgs(\"INFO\",container)\r\n", + "log_msgs(\"INFO\",SSISProject)\r\n", + "log_msgs(\"INFO\",'DRIVER='+f'{driver}'+';SERVER='+f'{server}'+';DATABASE='+f'{database}'+';UID='+f'{username}'+';PWD='+ f'{password}')\r\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.livy.statement-meta+json": { + "spark_pool": "notebookrun", + "session_id": 19, + "statement_id": 44, + "state": "finished", + "livy_statement_state": "cancelled", + "queued_time": "2021-08-26T01:02:54.91281Z", + "session_start_time": null, + "execution_start_time": "2021-08-26T01:02:56.1818556Z", + "execution_finish_time": "2021-08-26T01:03:27.445991Z" + }, + "text/plain": "StatementMeta(notebookrun, 19, 44, Finished, Cancelled)" + }, + "metadata": {} + } + ], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + "blob_connection_string = 'DefaultEndpointsProtocol=https;AccountName=' + blob_account_name + ';AccountKey=' + blob_account_key + \\\r\n", + "';EndpointSuffix=core.windows.net'\r\n", + "\r\n", + "execution_id_url = 'abfss://' + container + '@' + blob_account_name + '.dfs.core.windows.net/' + execution_id_path\r\n", + "\r\n", + "PARAM_Query = \\\r\n", + "\"SELECT CAST(a.[parameter_name] as nvarchar(max)) as parameter_name,CAST(a.parameter_value as nvarchar(max)) as parameter_value \\\r\n", + "FROM [SSISDB].[internal].[execution_parameter_values] a \\\r\n", + "INNER JOIN [SSISDB].[internal].[executions] b \\\r\n", + "ON ( a.execution_id = b.execution_id ) \\\r\n", + "WHERE [object_type] = 30 and a.[execution_id] = \\\r\n", + "(SELECT max([execution_id]) \\\r\n", + "from [SSISDB].[internal].[execution_parameter_values]) \\\r\n", + "AND a.[parameter_value] IS NOT NULL \\\r\n", + "AND a.[parameter_value] != 0 \\\r\n", + "AND b.[project_name] = '\" + f'{SSISProject}' + \"' \\\r\n", + "GROUP BY a.[object_type],a.[parameter_name],a.[parameter_value],b.[project_name],a.execution_id\"\r\n", + "\r\n", + "connection = pyodbc.connect('DRIVER='+f'{driver}'+';SERVER='+f'{server}'+';DATABASE='+f'{database}'+';UID='+f'{username}'+';PWD='+ f'{password}')" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.livy.statement-meta+json": { + "spark_pool": "notebookrun", + "session_id": 19, + "statement_id": 45, + "state": "finished", + "livy_statement_state": "available", + "queued_time": "2021-08-26T01:02:55.044096Z", + "session_start_time": null, + "execution_start_time": "2021-08-26T01:03:27.5401346Z", + "execution_finish_time": "2021-08-26T01:03:27.6873793Z" + }, + "text/plain": "StatementMeta(notebookrun, 19, 45, Finished, Available)" + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": "" + } + ], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + "Get_execution_id = \"SELECT CAST(MAX(a.execution_id) as int) as execution_id FROM [SSISDB].[internal].[execution_parameter_values] a\"\r\n", + "cursor = connection.cursor()\r\n", + "cursor.execute(Get_execution_id)\r\n", + "rows_check = cursor.fetchall()\r\n", + "columns_check = [column[0] for column in cursor.description]\r\n", + "if rows_check[0][0] is None:\r\n", + " New_Execution_Id = 0\r\n", + "else:\r\n", + " New_Execution_Id = int(rows_check[0][0]) " + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.livy.statement-meta+json": { + "spark_pool": "notebookrun", + "session_id": 19, + "statement_id": 46, + "state": "finished", + "livy_statement_state": "available", + "queued_time": "2021-08-26T01:02:55.133801Z", + "session_start_time": null, + "execution_start_time": "2021-08-26T01:03:27.7800084Z", + "execution_finish_time": "2021-08-26T01:03:59.0657852Z" + }, + "text/plain": "StatementMeta(notebookrun, 19, 46, Finished, Available)" + }, + "metadata": {} + }, + { + "output_type": "error", + "ename": "OperationalError", + "evalue": "('HYT00', '[HYT00] [Microsoft][ODBC Driver 17 for SQL Server]Login timeout expired (0) (SQLDriverConnect)')", + "traceback": [ + "OperationalError: ('HYT00', '[HYT00] [Microsoft][ODBC Driver 17 for SQL Server]Login timeout expired (0) (SQLDriverConnect)')", + "Traceback (most recent call last):\n", + "pyodbc.OperationalError: ('HYT00', '[HYT00] [Microsoft][ODBC Driver 17 for SQL Server]Login timeout expired (0) (SQLDriverConnect)')\n" + ] + } + ], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + "Old_Execution_Id = None\r\n", + "if New_Execution_Id != 0:\r\n", + " try:\r\n", + " df = spark.read.load(execution_id_url, format='csv', header=True).toPandas()\r\n", + " for index, row in df.iterrows():\r\n", + " row.execution_id\r\n", + " Old_Execution_Id = int(row.execution_id)\r\n", + " except:\r\n", + " pass\r\n", + "else:\r\n", + " df_Execution_Id = pd.DataFrame.from_records(rows_check, columns=columns_check)\r\n", + " print(df_Execution_Id)\r\n", + " if df_Execution_Id['execution_id'][0] is None:\r\n", + " Old_Execution_Id = None\r\n", + " New_Execution_Id = None\r\n", + " else:\r\n", + " df_sp_df_Execution_Id = spark.createDataFrame(df_Execution_Id)\r\n", + "\r\n", + " df_sp_df_Execution_Id.write.option('header', 'true').mode('overwrite').csv(execution_id_url)\r\n", + " Old_Execution_Id = 0\r\n", + "if (Old_Execution_Id is None and New_Execution_Id is not None) or (Old_Execution_Id < New_Execution_Id):\r\n", + " connection = pyodbc.connect('DRIVER='+f'{driver}'+';SERVER='+f'{server}'+';DATABASE='+f'{database}'+';UID='+f'{username}'+';PWD='+ f'{password}')\r\n", + " cursor = connection.cursor()\r\n", + " # Execute the query\r\n", + " cursor.execute(PARAM_Query)\r\n", + " rows = cursor.fetchall()\r\n", + " # # get column names from cursor\r\n", + " columns = [column[0] for column in cursor.description]\r\n", + " df_Params = pd.DataFrame.from_records(rows, columns=columns)\r\n", + " csvdata = df_Params.to_csv('Params.txt')\r\n", + " blob = BlobClient.from_connection_string(conn_str=f'{blob_connection_string}', container_name=f'{container}', blob_name=f'{blob_file_path}')\r\n", + " with open(\"Params.txt\", \"rb\") as data:\r\n", + " blob.upload_blob(data,overwrite=True)\r\n", + " \r\n", + " df_Execution_Id = pd.DataFrame.from_records(rows_check, columns=columns_check)\r\n", + " df_sp_df_Execution_Id = spark.createDataFrame(df_Execution_Id)\r\n", + " df_sp_df_Execution_Id.write.option('header', 'true').mode('overwrite').csv(execution_id_url)\r\n", + " mssparkutils.notebook.exit(True) \r\n", + "else: \r\n", + " mssparkutils.notebook.exit(False) \r\n", + "" + ] + } + ], + "metadata": { + "save_output": true, + "kernelspec": { + "name": "synapse_pyspark", + "display_name": "Synapse PySpark" + }, + "language_info": { + "name": "python" + }, + "synapse_widget": { + "version": "0.1", + "state": {} + } + } +} \ No newline at end of file diff --git a/examples/ssis/synapse/notebook/SSIS_Scan_Package.ipynb b/examples/ssis/synapse/notebook/SSIS_Scan_Package.ipynb new file mode 100644 index 0000000..11c8597 --- /dev/null +++ b/examples/ssis/synapse/notebook/SSIS_Scan_Package.ipynb @@ -0,0 +1,1223 @@ +{ + "nbformat": 4, + "nbformat_minor": 2, + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.livy.statement-meta+json": { + "spark_pool": "notebookrun", + "session_id": 13, + "statement_id": 75, + "state": "finished", + "livy_statement_state": "available", + "queued_time": "2021-08-24T09:57:08.0436386Z", + "session_start_time": null, + "execution_start_time": "2021-08-24T09:57:08.3071611Z", + "execution_finish_time": "2021-08-24T09:57:08.4562916Z" + }, + "text/plain": "StatementMeta(notebookrun, 13, 75, Finished, Available)" + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": "" + } + ], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true, + "tags": [ + "parameters" + ] + }, + "source": [ + "SSIS_AdlsAccountName = ''\r\n", + "SSIS_AdlsConatinerName = ''\r\n", + "SSIS_PackageFileName = ''\r\n", + "SISS_AppName = ''" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.livy.statement-meta+json": { + "spark_pool": "notebookrun", + "session_id": 22, + "statement_id": 2, + "state": "finished", + "livy_statement_state": "available", + "queued_time": "2021-08-24T14:37:57.6252587Z", + "session_start_time": null, + "execution_start_time": "2021-08-24T14:37:57.9137942Z", + "execution_finish_time": "2021-08-24T14:37:58.0601821Z" + }, + "text/plain": "StatementMeta(notebookrun, 22, 2, Finished, Available)" + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": "" + } + ], + "metadata": {}, + "source": [ + "import json\r\n", + "import re\r\n", + "import os\r\n", + "from enum import Enum\r\n", + "from pyspark.sql.utils import AnalysisException\r\n", + "from pyspark.sql.functions import explode\r\n", + "from pyspark.sql import SparkSession, Row\r\n", + "from notebookutils import mssparkutils\r\n", + "from pyspark.sql.functions import to_json" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.livy.statement-meta+json": { + "spark_pool": "notebookrun", + "session_id": 22, + "statement_id": 59, + "state": "finished", + "livy_statement_state": "available", + "queued_time": "2021-08-24T16:16:30.9420519Z", + "session_start_time": null, + "execution_start_time": "2021-08-24T16:16:31.0305166Z", + "execution_finish_time": "2021-08-24T16:16:31.5374748Z" + }, + "text/plain": "StatementMeta(notebookrun, 22, 59, Finished, Available)" + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": "" + } + ], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + "from pyspark.conf import SparkConf\r\n", + "from pyspark.sql import SparkSession\r\n", + "\r\n", + "my_jars = os.environ.get(\"SPARK_HOME\")\r\n", + "myconf = SparkConf()\r\n", + "myconf.set(\"spark.jars\",\"%s/jars/log4j-1.2.17.jar\" % my_jars)\r\n", + "spark = SparkSession\\\r\n", + " .builder\\\r\n", + " .appName(\"DB2_Test\")\\\r\n", + " .config(conf = myconf) \\\r\n", + " .getOrCreate()\r\n", + "\r\n", + "\r\n", + "Logger= spark._jvm.org.apache.log4j.Logger\r\n", + "mylogger = Logger.getLogger(SISS_AppName)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.livy.statement-meta+json": { + "spark_pool": "notebookrun", + "session_id": 22, + "statement_id": 60, + "state": "finished", + "livy_statement_state": "available", + "queued_time": "2021-08-24T16:16:33.2668085Z", + "session_start_time": null, + "execution_start_time": "2021-08-24T16:16:33.3528796Z", + "execution_finish_time": "2021-08-24T16:16:33.50672Z" + }, + "text/plain": "StatementMeta(notebookrun, 22, 60, Finished, Available)" + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": "INFO: Logger started!" + } + ], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + "def log_msgs(msg_type,msg):\r\n", + " \r\n", + " if msg_type.upper() == \"ERROR\":\r\n", + " print('ERROR: %s' % msg)\r\n", + " mylogger.error(msg)\r\n", + " elif msg_type.upper() == \"DEBUG\":\r\n", + " print('DEBUG: %s' % msg)\r\n", + " mylogger.debug(\"Fim\")\r\n", + " elif msg_type.upper() == \"TRACE\":\r\n", + " print('TRACE: %s' % msg)\r\n", + " mylogger.trace(\"Fim\")\r\n", + " else: # INFO\r\n", + " print('INFO: %s' % msg)\r\n", + " mylogger.info(\"Fim\")\r\n", + "\r\n", + "log_msgs('INFO', \"Logger started!\")\r\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.livy.statement-meta+json": { + "spark_pool": "notebookrun", + "session_id": 22, + "statement_id": 61, + "state": "finished", + "livy_statement_state": "available", + "queued_time": "2021-08-24T16:16:39.4396698Z", + "session_start_time": null, + "execution_start_time": "2021-08-24T16:16:39.5276584Z", + "execution_finish_time": "2021-08-24T16:16:39.6820178Z" + }, + "text/plain": "StatementMeta(notebookrun, 22, 61, Finished, Available)" + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": "" + } + ], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + "if SSIS_AdlsAccountName == '':\r\n", + " log_msgs('ERROR',\"SSIS_AdlsAccountName not configured\")\r\n", + "if SSIS_AdlsConatinerName == '':\r\n", + " log_msgs('ERROR',\"SSIS_AdlsConatinerName not configured\")\r\n", + "if SSIS_PackageFileName == '':\r\n", + " log_msgs('ERROR',\"SSIS_AdlsConatinerName not configured\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.livy.statement-meta+json": { + "spark_pool": "notebookrun", + "session_id": 22, + "statement_id": 62, + "state": "finished", + "livy_statement_state": "available", + "queued_time": "2021-08-24T16:16:42.7962527Z", + "session_start_time": null, + "execution_start_time": "2021-08-24T16:16:42.886389Z", + "execution_finish_time": "2021-08-24T16:16:43.0339289Z" + }, + "text/plain": "StatementMeta(notebookrun, 22, 62, Finished, Available)" + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": "" + } + ], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + "# Template names\r\n", + "package_template_nm = 'legacy_ssis_package_template.json'\r\n", + "process_template_nm = 'legacy_ssis_package_process_template.json'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.livy.statement-meta+json": { + "spark_pool": "notebookrun", + "session_id": 22, + "statement_id": 63, + "state": "finished", + "livy_statement_state": "available", + "queued_time": "2021-08-24T16:16:45.2955565Z", + "session_start_time": null, + "execution_start_time": "2021-08-24T16:16:45.3845649Z", + "execution_finish_time": "2021-08-24T16:16:45.5338549Z" + }, + "text/plain": "StatementMeta(notebookrun, 22, 63, Finished, Available)" + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": "" + } + ], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + "# Paths for SSIS files\r\n", + "package_path = 'abfss://' + SSIS_AdlsConatinerName + '@' + SSIS_AdlsAccountName + '.dfs.core.windows.net/pccsa_main/ssis-connector/working/' + SSIS_PackageFileName\r\n", + "param_path = 'abfss://' + SSIS_AdlsConatinerName + '@' + SSIS_AdlsAccountName + '.dfs.core.windows.net/pccsa_main/ssis-connector/ssis-package/pulled_params.txt'\r\n", + "connection_path = 'abfss://' + SSIS_AdlsConatinerName + '@' + SSIS_AdlsAccountName + '.dfs.core.windows.net/pccsa_main/ssis-connector/working/*.conmgr.json'\r\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.livy.statement-meta+json": { + "spark_pool": "notebookrun", + "session_id": 22, + "statement_id": 64, + "state": "finished", + "livy_statement_state": "available", + "queued_time": "2021-08-24T16:16:48.3676863Z", + "session_start_time": null, + "execution_start_time": "2021-08-24T16:16:48.4476977Z", + "execution_finish_time": "2021-08-24T16:16:48.6015108Z" + }, + "text/plain": "StatementMeta(notebookrun, 22, 64, Finished, Available)" + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": "" + } + ], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + "# Paths for Purview type templates and results storage\r\n", + "template_path = 'abfss://' + SSIS_AdlsConatinerName + '@' + SSIS_AdlsAccountName + '.dfs.core.windows.net/pccsa_main/ssis-connector/ssis-type-templates/'\r\n", + "result_blob_path = 'abfss://' + SSIS_AdlsConatinerName + '@' + SSIS_AdlsAccountName + '.dfs.core.windows.net/pccsa_main/incoming/results1.json'" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Pull SSIS Package" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.livy.statement-meta+json": { + "spark_pool": "notebookrun", + "session_id": 22, + "statement_id": 65, + "state": "finished", + "livy_statement_state": "available", + "queued_time": "2021-08-24T16:16:52.4938542Z", + "session_start_time": null, + "execution_start_time": "2021-08-24T16:16:52.5789689Z", + "execution_finish_time": "2021-08-24T16:16:57.8417395Z" + }, + "text/plain": "StatementMeta(notebookrun, 22, 65, Finished, Available)" + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": "" + } + ], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + "# Read in dataframes from json\r\n", + "package_df = spark.read.option(\"multiLine\", \"true\").load(package_path, format='json')\r\n", + "params_df = spark.read.load(param_path, format='csv',header = True,inferSchema = True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.livy.statement-meta+json": { + "spark_pool": "notebookrun", + "session_id": 22, + "statement_id": 66, + "state": "finished", + "livy_statement_state": "available", + "queued_time": "2021-08-24T16:16:55.9450246Z", + "session_start_time": null, + "execution_start_time": "2021-08-24T16:16:57.9267554Z", + "execution_finish_time": "2021-08-24T16:16:58.0740398Z" + }, + "text/plain": "StatementMeta(notebookrun, 22, 66, Finished, Available)" + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": "" + } + ], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + "def get_parameter_values(package: package_df, params: params_df):\r\n", + " # Retrieves name / value dictionary of parameters from SSIS package + parameters SSISDB output\r\n", + " try:\r\n", + " c_params = params_df.toJSON().collect()\r\n", + " d_param = {}\r\n", + " for param in c_params:\r\n", + " dict_p = json.loads(param)\r\n", + " param_name = dict_p[\"parameter_name\"]\r\n", + " if (param_name[0:3] == 'CM.'):\r\n", + " param_name = param_name[3:]\r\n", + " d_param[param_name] = dict_p[\"parameter_value\"]\r\n", + " except AnalysisException:\r\n", + " # This package seems to have no parameters\r\n", + " return {}\r\n", + " return d_param" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.livy.statement-meta+json": { + "spark_pool": "notebookrun", + "session_id": 22, + "statement_id": 67, + "state": "finished", + "livy_statement_state": "available", + "queued_time": "2021-08-24T16:17:01.6457008Z", + "session_start_time": null, + "execution_start_time": "2021-08-24T16:17:01.7326464Z", + "execution_finish_time": "2021-08-24T16:17:02.7791516Z" + }, + "text/plain": "StatementMeta(notebookrun, 22, 67, Finished, Available)" + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": "" + } + ], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + "# convert dataframes\r\n", + "package_json = json.loads(package_df.toJSON().collect()[0])\r\n", + "package_params = get_parameter_values(package_df, params_df)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Extract SSIS Package Data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.livy.statement-meta+json": { + "spark_pool": "notebookrun", + "session_id": 22, + "statement_id": 69, + "state": "finished", + "livy_statement_state": "available", + "queued_time": "2021-08-24T16:17:16.8942566Z", + "session_start_time": null, + "execution_start_time": "2021-08-24T16:17:16.9857144Z", + "execution_finish_time": "2021-08-24T16:17:17.1311613Z" + }, + "text/plain": "StatementMeta(notebookrun, 22, 69, Finished, Available)" + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": "'Package\\\\Data Flow Task\\\\Customers SQL MI'" + }, + "metadata": {} + }, + { + "output_type": "execute_result", + "data": {}, + "execution_count": 156, + "metadata": {} + } + ], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + "# Grouping string manipulations\r\n", + "# Examples:\r\n", + "# REFERENCE_ID: input = 'Package\\\\Data Flow Task\\\\Customers SQL MI.Inputs[OLE DB Destination Input]'\r\n", + "# output = 'Package\\\\Data Flow Task\\\\Customers SQL MI'\r\n", + "# PACKAGE_PROPERTIES: input = '@[$Package::BlobName]'\r\n", + "# output = 'BlobName'\r\n", + "# CONMGR_TYPE: input = 'Package.ConnectionManagers[SSIS Connection Manager for Azure Storage]'\r\n", + "# output = 'Package.'\r\n", + "# CONMGR_CON: input = 'Project.ConnectionManagers[purview-sqlmi.public.1ff9b19f84fa.database.windows.net,3342.purview-sqlmi-db.sqladmin]'\r\n", + "# output = 'purview-sqlmi.public.1ff9b19f84fa.database.windows.net,3342.purview-sqlmi-db.sqladmin'\r\n", + "\r\n", + "\r\n", + "class string_type (Enum):\r\n", + " PARAM_SEARCH = 0\r\n", + " REFERENCE_ID = 1\r\n", + " PACKAGE_PROPERTIES = 2\r\n", + " CONMGR_TYPE = 3\r\n", + " CONMGR_CON = 4\r\n", + "\r\n", + "\r\n", + "def transform_string(manip_type, t_string):\r\n", + " if manip_type == string_type.REFERENCE_ID:\r\n", + " return re.search('(.+)\\.',t_string).group(1)\r\n", + " if manip_type == string_type.PACKAGE_PROPERTIES:\r\n", + " return re.search('(?<=::)(.*)(?=\\])',t_string).group()\r\n", + " if manip_type == string_type.CONMGR_TYPE:\r\n", + " return re.search('.+\\.',t_string).group()\r\n", + " if manip_type == string_type.CONMGR_CON:\r\n", + " return re.search('\\[(.*?)\\]',t_string).group(1)\r\n", + "\r\n", + "# tests\r\n", + "transform_string(string_type.REFERENCE_ID, \"Package\\\\Data Flow Task\\\\Customers SQL MI.Inputs[OLE DB Destination Input]\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.livy.statement-meta+json": { + "spark_pool": "notebookrun", + "session_id": 22, + "statement_id": 70, + "state": "finished", + "livy_statement_state": "available", + "queued_time": "2021-08-24T16:17:26.2865182Z", + "session_start_time": null, + "execution_start_time": "2021-08-24T16:17:26.3851136Z", + "execution_finish_time": "2021-08-24T16:17:26.5308765Z" + }, + "text/plain": "StatementMeta(notebookrun, 22, 70, Finished, Available)" + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": "" + } + ], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + "def get_ssis_property(package, component, property_name, params):\r\n", + " prop_val = None\r\n", + " for prop in component[\"properties\"][\"property\"]:\r\n", + " if prop[\"@name\"] == property_name:\r\n", + " try:\r\n", + " prop_val = prop[\"_value_\"]\r\n", + " except KeyError:\r\n", + " # no value indicates that it is possibly an SSIS parameter check here\r\n", + " if params:\r\n", + " package_props = package[\"DTS:Executable\"][\"DTS:Executables\"][\"DTS:Executable\"][\"DTS:PropertyExpression\"]\r\n", + " for p in package_props:\r\n", + " if p[\"DTS:@Name\"] == \"[\" + component[\"@name\"] + \"].[\" + prop[\"@name\"] + \"]\":\r\n", + " prop_val = params[transform_string(string_type.PACKAGE_PROPERTIES,p[\"_value_\"])]\r\n", + " return prop_val" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.livy.statement-meta+json": { + "spark_pool": "notebookrun", + "session_id": 22, + "statement_id": 96, + "state": "finished", + "livy_statement_state": "available", + "queued_time": "2021-08-24T16:53:17.9963205Z", + "session_start_time": null, + "execution_start_time": "2021-08-24T16:53:18.1140127Z", + "execution_finish_time": "2021-08-24T16:53:18.2640334Z" + }, + "text/plain": "StatementMeta(notebookrun, 22, 96, Finished, Available)" + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": "{'Data Source': '20.206.80.219', 'User ID': 'vmadmin', 'Initial Catalog': 'purview-sqlmi-db', 'Provider': 'SQLNCLI11.1', 'Auto Translate': 'False'}" + } + ], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + "def connection_string_to_dictionary(constr):\r\n", + " output = {}\r\n", + " nm_vals = constr.split(';')\r\n", + " for nm_val in nm_vals:\r\n", + " if nm_val != '':\r\n", + " list_nm_and_val = nm_val.split('=')\r\n", + " output[list_nm_and_val[0]] = list_nm_and_val[1]\r\n", + " return output" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.livy.statement-meta+json": { + "spark_pool": "notebookrun", + "session_id": 22, + "statement_id": 102, + "state": "finished", + "livy_statement_state": "available", + "queued_time": "2021-08-24T17:16:05.8038825Z", + "session_start_time": null, + "execution_start_time": "2021-08-24T17:16:05.9081798Z", + "execution_finish_time": "2021-08-24T17:16:06.0540887Z" + }, + "text/plain": "StatementMeta(notebookrun, 22, 102, Finished, Available)" + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": "" + } + ], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + "def get_mappings(package, params):\r\n", + " # package = package_json\r\n", + " # params = package_params\r\n", + " ds_map = {}\r\n", + " ds_map[\"DatasetMapping\"] = {}\r\n", + " src_clmn = '*'\r\n", + "\r\n", + " connections = []\r\n", + "\r\n", + " components = package[\"DTS:Executable\"][\"DTS:Executables\"][\"DTS:Executable\"][\"DTS:ObjectData\"]\\\r\n", + " [\"pipeline\"][\"components\"][\"component\"]\r\n", + "\r\n", + " # Need to loop through paths and figure out how to create multiple outputs in lineage\r\n", + "\r\n", + " for component in components:\r\n", + " conmgrRefId = component[\"connections\"][\"connection\"][\"@connectionManagerRefId\"]\r\n", + " if transform_string(string_type.CONMGR_TYPE,conmgrRefId) == \"Package.\":\r\n", + " connection = transform_string(string_type.CONMGR_CON,conmgrRefId)\r\n", + " if connection + '.ConnectionString' in params:\r\n", + " conStr = params[connection + '.ConnectionString']\r\n", + " else:\r\n", + " for con in package[\"DTS:Executable\"][\"DTS:ConnectionManagers\"][\"DTS:ConnectionManager\"]:\r\n", + " if con[\"DTS:@refId\"] == conmgrRefId:\r\n", + " if con[\"DTS:@CreationName\"] == \"OLEDB\":\r\n", + " conStr = con[\"DTS:ObjectData\"][\"DTS:ConnectionManager\"][\"DTS:@ConnectionString\"]\r\n", + " elif con[\"DTS:@CreationName\"] == \"AzureStorage\":\r\n", + " conStr = con[\"DTS:ObjectData\"][\"AzureStorageConnectionManager\"][\"@ConnectionString\"]\r\n", + " else:\r\n", + " conStr = transform_string(string_type.CONMGR_CON, conmgrRefId)\r\n", + " if component[\"@name\"] == \"Azure Blob Source\":\r\n", + " blob_name = get_ssis_property(package_json, component, \"Blob Name\", params)\r\n", + " blob_container = get_ssis_property(package_json, component, \"Blob Container\", params)\r\n", + " # returns a dictionary e.g. input = \"AccountName=purviewaccdl;DefaultEndpointsProtocol=https\" output = {'AccountName':'purviewaccdl','DefaultEndpointsProtocol':'https'}\r\n", + " con_props = dict(x.split(\"=\") for x in re.split(\";\",conStr[:-1]))\r\n", + " ds_map[\"DatasetMapping\"][\"Source\"] = con_props[\"DefaultEndpointsProtocol\"] + \"://\" + con_props[\"AccountName\"] + \".blob.\" + con_props[\"EndpointSuffix\"] + \"/\" \\\r\n", + " + blob_container + \"/\" + blob_name.strip()\r\n", + " #ds_map[\"DatasetMapping\"][\"Source\"] = src_clmn\r\n", + " if component[\"@name\"] == \"OLE DB Destination\":\r\n", + " tbl = params[\"Table\"]\r\n", + " tbl = tbl.replace('[', '')\r\n", + " tbl = tbl.replace(']', '')\r\n", + " tbl = tbl.replace('.', '/')\r\n", + " con_dict = connection_string_to_dictionary(conStr)\r\n", + " source_string = \"mssql://\" + con_dict['Data Source'] + \"/\" + con_dict['Initial Catalog'] + \"/\" + tbl\r\n", + " ds_map[\"DatasetMapping\"][\"Sink\"] = source_string\r\n", + " ds_map[\"ColumnMapping\"] = []\r\n", + " columns = component[\"inputs\"][\"input\"][\"inputColumns\"][\"inputColumn\"]\r\n", + " for col in columns:\r\n", + " src_col = col[\"@cachedName\"]\r\n", + " dst_col = col[\"@lineageId\"]\r\n", + " dst_col = dst_col[dst_col.rfind('[')+1:-1]\r\n", + " ds_map[\"ColumnMapping\"].append({\"Source\": src_col,\"Sink\": dst_col})\r\n", + "\r\n", + " return [ds_map]\r\n", + " # print(ds_map)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Transform Purview Templates" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.livy.statement-meta+json": { + "spark_pool": "notebookrun", + "session_id": 22, + "statement_id": 85, + "state": "finished", + "livy_statement_state": "available", + "queued_time": "2021-08-24T16:29:26.8679541Z", + "session_start_time": null, + "execution_start_time": "2021-08-24T16:29:26.9528994Z", + "execution_finish_time": "2021-08-24T16:29:29.6929179Z" + }, + "text/plain": "StatementMeta(notebookrun, 22, 85, Finished, Available)" + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": "" + } + ], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": false + }, + "source": [ + "# Get output templates\r\n", + "purview_package_df = spark.read.option(\"multiLine\", \"true\").load(template_path + package_template_nm, format='json')\r\n", + "purview_package_process_df = spark.read.option(\"multiLine\", \"true\").load(template_path + process_template_nm, format='json')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.livy.statement-meta+json": { + "spark_pool": "notebookrun", + "session_id": 22, + "statement_id": 86, + "state": "finished", + "livy_statement_state": "available", + "queued_time": "2021-08-24T16:29:44.5765558Z", + "session_start_time": null, + "execution_start_time": "2021-08-24T16:29:44.6635822Z", + "execution_finish_time": "2021-08-24T16:29:45.7066175Z" + }, + "text/plain": "StatementMeta(notebookrun, 22, 86, Finished, Available)" + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": "" + } + ], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + "# convert DataFrames to JSON objects\r\n", + "purview_package_json = json.loads(purview_package_df.toJSON().collect()[0])\r\n", + "purview_package_process_json = json.loads(purview_package_process_df.toJSON().collect()[0])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.livy.statement-meta+json": { + "spark_pool": "notebookrun", + "session_id": 22, + "statement_id": 103, + "state": "finished", + "livy_statement_state": "available", + "queued_time": "2021-08-24T17:16:16.1818383Z", + "session_start_time": null, + "execution_start_time": "2021-08-24T17:16:16.2725055Z", + "execution_finish_time": "2021-08-24T17:16:16.4231984Z" + }, + "text/plain": "StatementMeta(notebookrun, 22, 103, Finished, Available)" + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": "" + } + ], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + "def transform(source, process_template, package_template):\r\n", + " # Note: this 'hello world' example is designed for clarity and easy customization in a PoC / MVP. \r\n", + " # ToDo: Make these pipeline variables \r\n", + " package_type = \"legacy_ssis_package\"\r\n", + " package_process_type = \"legacy_ssis_package_process\"\r\n", + " package_fqn = \"ssisleg://ssis_legacy.net/adflineage/PurviewSSISTest/SSIS_Movie_Loader_package_3\"\r\n", + " package_process_fqn = \"ssisleg://ssis_legacy.net/adflineage/PurviewSSISTest/Data_Flow_Task\"\r\n", + "\r\n", + " # source FQN, destination FQN, Column mappings string\r\n", + " in_out_map = get_mappings(package_json, package_params)[0]\r\n", + "\r\n", + " # Apply mappings legacy_ssis_package_mapping\r\n", + " package_template[\"attributes\"][\"name\"] = source[\"DTS:Executable\"][\"DTS:@ObjectName\"]\r\n", + " package_template[\"attributes\"][\"description\"] = source[\"DTS:Executable\"][\"DTS:@ExecutableType\"]\r\n", + " package_template[\"attributes\"][\"description\"] = source[\"DTS:Executable\"][\"DTS:@ExecutableType\"]\r\n", + " package_template[\"attributes\"][\"qualifiedName\"] = package_fqn\r\n", + " package_template[\"typeName\"] = package_type\r\n", + "\r\n", + " # Apply mappings legacy_ssis_package_process_mapping\r\n", + " process_template[\"attributes\"][\"qualifiedName\"] = package_process_fqn\r\n", + " process_template[\"attributes\"][\"name\"] = source[\"DTS:Executable\"][\"DTS:Executables\"][\"DTS:Executable\"][\"DTS:@ObjectName\"]\r\n", + " process_template[\"attributes\"][\"inputs\"][0][\"uniqueAttributes\"][\"qualifiedName\"] = in_out_map[\"DatasetMapping\"][\"Source\"]\r\n", + " process_template[\"attributes\"][\"outputs\"][0][\"uniqueAttributes\"][\"qualifiedName\"] = in_out_map[\"DatasetMapping\"][\"Sink\"]\r\n", + " process_template[\"relationshipAttributes\"][\"inputs\"][0][\"displayText\"] = source[\"DTS:Executable\"][\"DTS:Executables\"][\"DTS:Executable\"][\"DTS:ObjectData\"][\"pipeline\"][\"components\"][\"component\"][0][\"@description\"]\r\n", + " process_template[\"relationshipAttributes\"][\"inputs\"][0][\"qualifiedName\"] = in_out_map[\"DatasetMapping\"][\"Source\"]\r\n", + " process_template[\"relationshipAttributes\"][\"outputs\"][0][\"displayText\"] = source[\"DTS:Executable\"][\"DTS:Executables\"][\"DTS:Executable\"][\"DTS:ObjectData\"][\"pipeline\"][\"components\"][\"component\"][1][\"@description\"]\r\n", + " process_template[\"relationshipAttributes\"][\"outputs\"][0][\"qualifiedName\"] = in_out_map[\"DatasetMapping\"][\"Sink\"]\r\n", + " process_template[\"relationshipAttributes\"][\"parent\"][\"displayText\"] = source[\"DTS:Executable\"][\"DTS:Executables\"][\"DTS:Executable\"][\"DTS:@refId\"]\r\n", + " # Need to substitue '*' for source - todo change this\r\n", + " mystr = json.dumps([in_out_map])\r\n", + " mystr = mystr.replace(\"https://purviewaccdl.blob.core.windows.net/sourcefiles/Movies.csv\", \"*\")\r\n", + "\r\n", + " process_template[\"attributes\"][\"columnMapping\"] = mystr\r\n", + " #process_template[\"attributes\"][\"columnMapping\"] = json.dumps([in_out_map])\r\n", + " process_template[\"typeName\"] = package_process_type\r\n", + "\r\n", + "\r\n", + " return package_template, process_template\r\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.livy.statement-meta+json": { + "spark_pool": "notebookrun", + "session_id": 22, + "statement_id": 106, + "state": "finished", + "livy_statement_state": "available", + "queued_time": "2021-08-24T17:16:57.2663545Z", + "session_start_time": null, + "execution_start_time": "2021-08-24T17:16:57.3582392Z", + "execution_finish_time": "2021-08-24T17:16:57.5010675Z" + }, + "text/plain": "StatementMeta(notebookrun, 22, 106, Finished, Available)" + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": "" + } + ], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + "# run the transformation then package up the results to write to file\r\n", + "package_transformed, package_process_transformed = transform(package_json,purview_package_process_json,purview_package_json)\r\n", + "concat_transform = [package_transformed,package_process_transformed]\r\n", + "concat_transform_str = json.dumps(concat_transform)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.livy.statement-meta+json": { + "spark_pool": "notebookrun", + "session_id": 22, + "statement_id": 107, + "state": "finished", + "livy_statement_state": "available", + "queued_time": "2021-08-24T17:17:00.1536502Z", + "session_start_time": null, + "execution_start_time": "2021-08-24T17:17:00.2532469Z", + "execution_finish_time": "2021-08-24T17:17:00.4109338Z" + }, + "text/plain": "StatementMeta(notebookrun, 22, 107, Finished, Available)" + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": "TRACE: [{\"attributes\": {\"description\": \"Microsoft.Package\", \"lastRunTime\": \"\", \"name\": \"Package\", \"qualifiedName\": \"ssisleg://ssis_legacy.net/adflineage/PurviewSSISTest/SSIS_Movie_Loader_package_3\"}, \"typeName\": \"legacy_ssis_package\"}, {\"attributes\": {\"columnMapping\": \"[{\\\"DatasetMapping\\\": {\\\"Source\\\": \\\"https://pccsastorage2732136.blob.core.windows.net/pccsa/pccsa_main/ssis-connector/example-data/MovCustomers.csv\\\", \\\"Sink\\\": \\\"mssql://20.206.80.219/purview-sqlmi-db/dbo/MovCustomers\\\"}, \\\"ColumnMapping\\\": [{\\\"Source\\\": \\\"Customer_ID\\\", \\\"Sink\\\": \\\"Customer_ID\\\"}, {\\\"Source\\\": \\\"Last_Name\\\", \\\"Sink\\\": \\\"Last_Name\\\"}, {\\\"Source\\\": \\\"First_Name\\\", \\\"Sink\\\": \\\"First_Name\\\"}, {\\\"Source\\\": \\\"Addr_1\\\", \\\"Sink\\\": \\\"Addr_1\\\"}, {\\\"Source\\\": \\\"Addr_2\\\", \\\"Sink\\\": \\\"Addr_2\\\"}, {\\\"Source\\\": \\\"City\\\", \\\"Sink\\\": \\\"City\\\"}, {\\\"Source\\\": \\\"State\\\", \\\"Sink\\\": \\\"State\\\"}, {\\\"Source\\\": \\\"Zip_Code\\\", \\\"Sink\\\": \\\"Zip_Code\\\"}, {\\\"Source\\\": \\\"Phone_Number\\\", \\\"Sink\\\": \\\"Phone_Number\\\"}, {\\\"Source\\\": \\\"Created_Date\\\", \\\"Sink\\\": \\\"Created_Date\\\"}, {\\\"Source\\\": \\\"Updated_Date\\\", \\\"Sink\\\": \\\"Updated_Date\\\"}]}]\", \"inputs\": [{\"uniqueAttributes\": {\"qualifiedName\": \"https://pccsastorage2732136.blob.core.windows.net/pccsa/pccsa_main/ssis-connector/example-data/MovCustomers.csv\"}}], \"name\": \"Data Flow Task\", \"outputs\": [{\"uniqueAttributes\": {\"qualifiedName\": \"mssql://20.206.80.219/purview-sqlmi-db/dbo/MovCustomers\"}}], \"qualifiedName\": \"ssisleg://ssis_legacy.net/adflineage/PurviewSSISTest/Data_Flow_Task\"}, \"relationshipAttributes\": {\"inputs\": [{\"displayText\": \"Extracts data from a blob in Azure Storage. The supported file formats are: Text and Avro. To work with Azure Data Lake Storage Gen2, use Flexible File Source with Azure Storage Connection Manager instead.\", \"entityStatus\": \"ACTIVE\", \"qualifiedName\": \"https://pccsastorage2732136.blob.core.windows.net/pccsa/pccsa_main/ssis-connector/example-data/MovCustomers.csv\"}], \"outputs\": [{\"displayText\": \"OLE DB Destination\", \"entityStatus\": \"ACTIVE\", \"qualifiedName\": \"mssql://20.206.80.219/purview-sqlmi-db/dbo/MovCustomers\"}], \"parent\": {\"displayText\": \"Package\\\\Data Flow Task\", \"relationshipAttributes\": {\"typeName\": \"legacy_ssis_package_process_parent\"}, \"relationshipType\": \"legacy_ssis_package_process_parent\", \"typeName\": \"legacy_ssis_package\"}}, \"status\": \"ACTIVE\", \"typeName\": \"legacy_ssis_package_process\"}]" + } + ], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + "# Test Output\r\n", + "log_msgs('TRACE', concat_transform_str)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Copy JSON to ADLS" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.livy.statement-meta+json": { + "spark_pool": "notebookrun", + "session_id": 22, + "statement_id": 108, + "state": "finished", + "livy_statement_state": "available", + "queued_time": "2021-08-24T17:17:10.1737554Z", + "session_start_time": null, + "execution_start_time": "2021-08-24T17:17:10.25911Z", + "execution_finish_time": "2021-08-24T17:17:13.0213465Z" + }, + "text/plain": "StatementMeta(notebookrun, 22, 108, Finished, Available)" + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": "True" + }, + "metadata": {} + }, + { + "output_type": "execute_result", + "data": {}, + "execution_count": 195, + "metadata": {} + } + ], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + "# Write result file\r\n", + "mssparkutils.fs.put(result_blob_path, concat_transform_str, overwrite=True)" + ] + } + ], + "metadata": { + "kernelspec": { + "name": "synapse_pyspark", + "display_name": "Synapse PySpark" + }, + "language_info": { + "name": "python" + } + } +} \ No newline at end of file diff --git a/examples/ssis/synapse/pipeline/SSIS_Package_Pipeline.json b/examples/ssis/synapse/pipeline/SSIS_Package_Pipeline.json new file mode 100644 index 0000000..c05758f --- /dev/null +++ b/examples/ssis/synapse/pipeline/SSIS_Package_Pipeline.json @@ -0,0 +1,505 @@ +{ + "name": "SSIS_Package_Pipeline", + "properties": { + "activities": [ + { + "name": "SSIS Parameters", + "type": "SynapseNotebook", + "dependsOn": [ + { + "activity": "Set SQL Con Str", + "dependencyConditions": [ + "Succeeded" + ] + }, + { + "activity": "Set ADLS Key", + "dependencyConditions": [ + "Succeeded" + ] + }, + { + "activity": "Set SQL PW", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "notebook": { + "referenceName": "SSISDB_Get_Params", + "type": "NotebookReference" + }, + "parameters": { + "blob_file_path": { + "value": "pccsa_main/ssis-connector/ssis-package/pulled_params.txt", + "type": "string" + }, + "server": { + "value": ",1433", + "type": "string" + }, + "database": { + "value": "purview-sqlmi-db", + "type": "string" + }, + "username": { + "value": "", + "type": "string" + }, + "password": { + "value": { + "value": "@variables('SQL-PW')", + "type": "Expression" + }, + "type": "string" + }, + "driver": { + "value": "{ODBC Driver 17 for SQL Server}", + "type": "string" + }, + "container": { + "value": "pccsa", + "type": "string" + }, + "blob_account_name": { + "value": "", + "type": "string" + }, + "blob_account_key": { + "value": { + "value": "@variables('ADLS-Key')", + "type": "Expression" + }, + "type": "string" + }, + "SSISProject": { + "value": "PurviewSSISTest", + "type": "string" + }, + "execution_id_path": { + "value": "pccsa_main/ssis-connector/execution-id/", + "type": "string" + } + }, + "snapshot": true + } + }, + { + "name": "if_new_package", + "type": "IfCondition", + "dependsOn": [ + { + "activity": "SSIS Parameters", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "expression": { + "value": "@activity('SSIS Parameters').output.status.Output.result.exitValue", + "type": "Expression" + }, + "ifTrueActivities": [ + { + "name": "SSIS Package transform to JSON", + "type": "Copy", + "dependsOn": [ + { + "activity": "PowerShell_WebHook", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "XmlSource", + "storeSettings": { + "type": "AzureBlobFSReadSettings", + "recursive": true, + "wildcardFolderPath": "pccsa_main/ssis-connector/ssis-package", + "wildcardFileName": "*.dtsx", + "enablePartitionDiscovery": false + }, + "formatSettings": { + "type": "XmlReadSettings", + "validationMode": "none", + "namespaces": true + } + }, + "sink": { + "type": "JsonSink", + "storeSettings": { + "type": "AzureBlobFSWriteSettings" + }, + "formatSettings": { + "type": "JsonWriteSettings" + } + }, + "enableStaging": false + }, + "inputs": [ + { + "referenceName": "Xml1", + "type": "DatasetReference" + } + ], + "outputs": [ + { + "referenceName": "Json1", + "type": "DatasetReference" + } + ] + }, + { + "name": "SSIS Connectors transform to JSON", + "type": "Copy", + "dependsOn": [ + { + "activity": "PowerShell_WebHook", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "XmlSource", + "storeSettings": { + "type": "AzureBlobFSReadSettings", + "recursive": true, + "wildcardFolderPath": "pccsa_main/ssis-connector/ssis-package", + "wildcardFileName": "*.conmgr", + "enablePartitionDiscovery": false + }, + "formatSettings": { + "type": "XmlReadSettings", + "validationMode": "none", + "namespaces": true + } + }, + "sink": { + "type": "JsonSink", + "storeSettings": { + "type": "AzureBlobFSWriteSettings" + }, + "formatSettings": { + "type": "JsonWriteSettings" + } + }, + "enableStaging": false + }, + "inputs": [ + { + "referenceName": "Xml2", + "type": "DatasetReference" + } + ], + "outputs": [ + { + "referenceName": "Json1", + "type": "DatasetReference" + } + ] + }, + { + "name": "PowerShell_WebHook", + "type": "WebHook", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "url": "", + "method": "POST", + "body": { + "value": "@json(concat('{\n\"StorageAccountName\":\"', pipeline().parameters.StorageAccountName, '\", \n\"FileSystemName\":\"', pipeline().parameters.FileSystemName,'\",\n\"FolderName\":\"', pipeline().parameters.FolderName, '\",\n\"ProjectName\":\"', pipeline().parameters.ProjectName, '\",\n\"VaultName\":\"', pipeline().parameters.VaultName, '\",\n\"ADLSecretName\":\"', variables('ADLS-Key') ,'\",\n\"SQLSecretName\":\"', variables('SQL-Conn-Str'),'\",\n\"dirname\":\"', pipeline().parameters.dirname, '\",\n}') )", + "type": "Expression" + }, + "timeout": "00:02:30", + "reportStatusOnCallBack": true + } + }, + { + "name": "SSIS Scan", + "type": "SynapseNotebook", + "dependsOn": [ + { + "activity": "SSIS Package transform to JSON", + "dependencyConditions": [ + "Succeeded" + ] + }, + { + "activity": "SSIS Connectors transform to JSON", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "notebook": { + "referenceName": "SSIS_Scan_Package", + "type": "NotebookReference" + }, + "parameters": { + "SSIS_AdlsAccountName": { + "value": { + "value": "@pipeline().parameters.StorageAccountName", + "type": "Expression" + }, + "type": "string" + }, + "SSIS_AdlsConatinerName": { + "value": { + "value": "@pipeline().parameters.FileSystemName", + "type": "Expression" + }, + "type": "string" + }, + "SSIS_PackageFileName": { + "value": { + "value": "Package.dtsx.json", + "type": "Expression" + }, + "type": "string" + }, + "SISS_AppName": { + "value": { + "value": "@pipeline().parameters.ProjectName", + "type": "Expression" + }, + "type": "string" + } + }, + "snapshot": true + } + } + ] + } + }, + { + "name": "KV Get SQL Connection String", + "type": "WebActivity", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "url": "?api-version=7.0", + "connectVia": { + "referenceName": "AutoResolveIntegrationRuntime", + "type": "IntegrationRuntimeReference" + }, + "method": "GET", + "authentication": { + "type": "MSI", + "resource": "https://vault.azure.net" + } + } + }, + { + "name": "KV Get Storage Account Key", + "type": "WebActivity", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "url": "?api-version=7.0", + "connectVia": { + "referenceName": "AutoResolveIntegrationRuntime", + "type": "IntegrationRuntimeReference" + }, + "method": "GET", + "authentication": { + "type": "MSI", + "resource": "https://vault.azure.net" + } + } + }, + { + "name": "KV Get SQL Password", + "type": "WebActivity", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "url": "?api-version=7.0", + "connectVia": { + "referenceName": "AutoResolveIntegrationRuntime", + "type": "IntegrationRuntimeReference" + }, + "method": "GET", + "authentication": { + "type": "MSI", + "resource": "https://vault.azure.net" + } + } + }, + { + "name": "Set SQL Con Str", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "KV Get SQL Connection String", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "variableName": "SQL-Conn-Str", + "value": { + "value": "@activity('KV Get SQL Connection String').output.value", + "type": "Expression" + } + } + }, + { + "name": "Set ADLS Key", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "KV Get Storage Account Key", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "variableName": "ADLS-Key", + "value": { + "value": "@activity('KV Get Storage Account Key').output.value", + "type": "Expression" + } + } + }, + { + "name": "Set SQL PW", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "KV Get SQL Password", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "variableName": "SQL-PW", + "value": { + "value": "@activity('KV Get SQL Password').output.value", + "type": "Expression" + } + } + } + ], + "parameters": { + "StorageAccountName": { + "type": "string", + "defaultValue": "" + }, + "FileSystemName": { + "type": "string", + "defaultValue": "pccsa" + }, + "FolderName": { + "type": "string", + "defaultValue": "ssistest" + }, + "ProjectName": { + "type": "string", + "defaultValue": "PurviewSSISTest" + }, + "VaultName": { + "type": "string", + "defaultValue": "" + }, + "ADLSecretName": { + "type": "string", + "defaultValue": "storage_account_key" + }, + "SQLSecretName": { + "type": "string", + "defaultValue": "SQL-Connection-String" + }, + "dirname": { + "type": "string", + "defaultValue": "pccsa_main/ssis-connector/ssis-package/" + } + }, + "variables": { + "IsNewSsisPkg": { + "type": "Boolean" + }, + "SQL-Conn-Str": { + "type": "String" + }, + "ADLS-Key": { + "type": "String" + }, + "SQL-PW": { + "type": "String" + } + }, + "folder": { + "name": "SSIS Example" + }, + "annotations": [], + "lastPublishTime": "2021-05-17T23:19:40Z" + }, + "type": "Microsoft.Synapse/workspaces/pipelines" +} \ No newline at end of file diff --git a/examples/tag_db/deploy/deploy_tag_db.md b/examples/tag_db/deploy/deploy_tag_db.md new file mode 100644 index 0000000..680176c --- /dev/null +++ b/examples/tag_db/deploy/deploy_tag_db.md @@ -0,0 +1,24 @@ +# Deploy Tag Database Source Example + +## Components + +![PCCSA Block Diagram](../../../assets/images/osi_pi_deploy_blocks.svg) + +## Prerequisite + +Follow the instructions for deploying the base solution under [purview_connector_services](../../../purview_connector_services/deploy/deploy_sa.md) + +## Run Installation Script + +* start the cloud CLI in bash mode +* cd to the cloud storage directory (clouddrive) +* navigate to the PurviewACC/examples/tag_db/deploy directory +* run './deploy_tag_db.sh' + +## Reference - script actions + +* Sets up the blob storage directory structure for the Tag DB example +* In Synapse + * Imports dataset definitions + * Imports the notebook + * Imports the pipeline definitions diff --git a/examples/tag_db/deploy/deploy_tag_db.sh b/examples/tag_db/deploy/deploy_tag_db.sh new file mode 100644 index 0000000..c0170e2 --- /dev/null +++ b/examples/tag_db/deploy/deploy_tag_db.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# dynamically load missing az dependancies without prompting +az config set extension.use_dynamic_install=yes_without_prompt --output none + +# This script requires contributor and user access administrator permissions to run +source ../../../purview_connector_services/deploy/export_names.sh + +# To run in Azure Cloud CLI, comment this section out +# az login --output none +# az account set --subscription "Early Access Engineering Subscription" --output none + +# Upload storage folders and data +# Create storage dir structure +echo "Create storage folders" +mkdir tag-db-connector; cd tag-db-connector; mkdir tag-db-json; touch ./tag-db-json/tmp; mkdir tag-db-processed;\ +touch ./tag-db-processed/tmp; mkdir tag-db-purview-json; touch ./tag-db-purview-json/tmp; mkdir tag-db-xml;\ +touch ./tag-db-xml/tmp; cd .. +# Upload dir structure to storage +az storage fs directory upload -f pccsa --account-name $storage_name -s ./tag-db-connector -d ./pccsa_main --recursive --output none +# Remove tmp directory structure +rm -r tag-db-connector + +# Upload sample data +az storage fs file upload -f pccsa --account-name $storage_name -s ../example_data/tag-db-xml-sample.xml -p ./pccsa_main/tag-db-connector/tag-db-xml/tag-db-xml-sample.xml --auth-mode login + +# Configure Synapse Notebooks and Pipelines + +# Data set +echo "Creating TAG_DB Datasets" +az synapse dataset create --workspace-name $synapse_name --name TAG_DB_JSON --file @../synapse/dataset/TAG_DB_JSON.json +az synapse dataset create --workspace-name $synapse_name --name TAG_DB_Metadata_XML --file @../synapse/dataset/TAG_DB_Metadata_XML.json + +# Notebook +echo "Creating Purview TAG_DB Scan Notebook" +az synapse notebook create --workspace-name $synapse_name --spark-pool-name notebookrun --name Purview_TAG_DB_Scan --file @../synapse/notebook/Purview_TAG_DB_Scan.ipynb + +# Pipeline +echo "Creating TAG_DB pipeline" +# Build substitutions for pipeline json - note sed can use any delimeter so url changed to avoid conflict with slash char +pipeline_sub="s@@$storage_name@" +sed $pipeline_sub '../synapse/pipeline/Converte TAG DB XML Metadata to Json.json' > '../synapse/pipeline/Converte TAG DB XML Metadata to Json-tmp.json' +# Create pipeline in Synapse +az synapse pipeline create --workspace-name $synapse_name --name 'Converte TAG DB XML Metadata to Json' --file '@../synapse/pipeline/Converte TAG DB XML Metadata to Json-tmp.json' --debug +# Delete the tmp json +rm '../synapse/pipeline/Converte TAG DB XML Metadata to Json-tmp.json' \ No newline at end of file diff --git a/examples/tag_db/example_data/something.csv b/examples/tag_db/example_data/something.csv new file mode 100644 index 0000000..e69de29 diff --git a/examples/tag_db/example_data/tag-db-xml-sample.xml b/examples/tag_db/example_data/tag-db-xml-sample.xml new file mode 100644 index 0000000..75e19ee --- /dev/null +++ b/examples/tag_db/example_data/tag-db-xml-sample.xml @@ -0,0 +1,821 @@ + + + + DB_Operational + Operational Database for monitoring Specific operational Area + + DefaultPIServer + name_of_the_pi_server + + + DefaultPIServerID + 12345678910 + + + Company_name + Root Element + + false + DOMAIN\username_02 + + 1970-01-01T00:00:00Z + 9999-12-31T23:59:59Z + + Area01 + Element Level 02 + + false + DOMAIN\username_02 + + 1970-01-01T00:00:00Z + 9999-12-31T23:59:59Z + + Country + Element Level 3 + + false + DOMAIN\username + + 1970-01-01T00:00:00Z + 9999-12-31T23:59:59Z + + Location 01 + Element Level 4 + + false + DOMAIN\USER_03 + + 1970-01-01T00:00:00Z + 9999-12-31T23:59:59Z + + Attribute 01 + + false + false + false + false + + + -5 + String + + + + + + Attribute 02 + + false + false + false + false + + + -5 + String + + String Builder + %Element%&#xD; + + + Attribute 02 + + false + false + false + false + + + 2 + Decimal + + + + + + Sub Location 01 + + + false + DOMAIN\username + + 1970-01-01T00:00:00Z + 9999-12-31T23:59:59Z + Category Type 01 + + Alarme + + false + false + false + false + + + -5 + Double + + Good + 1970-01-01T00:00:00Z + 17 + + + + + Group + + false + false + false + false + + + -5 + AFEnumerationValue + EnumerationSets[Goup 01] + + + + + + Controller + + false + false + false + false + + + -5 + AFEnumerationValue + EnumerationSets[On/Off] + + + + + + Availibility Average + + false + false + false + false + + % + -5 + Double + + PI Point + \\name_of_the_pi_server?server_id\Location 03.Availibility Average?43257 + + + Count + + false + false + false + false + + + -5 + Double + + Good + 1970-01-01T00:00:00Z + 145 + + + + + Name + + false + false + false + false + + + -5 + String + + String Builder + %Element% + + + Last Level 01 + + + false + DOMAIN\username + + 1970-01-01T00:00:00Z + 9999-12-31T23:59:59Z + + Alarme + + false + false + false + false + + + -5 + Double + + Good + 1970-01-01T00:00:00Z + 17 + + + + + Group + + false + false + false + false + + + -5 + AFEnumerationValue + EnumerationSets[Goup 01] + + + + + + Controller + + false + false + false + false + + + -5 + AFEnumerationValue + EnumerationSets[On/Off] + + + + + + Availibility Average + + false + false + false + false + + % + -5 + Double + + PI Point + \\name_of_the_pi_server?server_id\Location 03.Availibility Average?43257 + + + Count + + false + false + false + false + + + -5 + Double + + Good + 1970-01-01T00:00:00Z + 145 + + + + + Name + + false + false + false + false + + + -5 + String + + String Builder + %Element% + + + Availibility Calc + + + + + Enabled + false + High + 0 + 0 + + Company_name\Area01\Country\Location 02\Location 03\Element 01 + + Administrators:A(r,w,rd,wd,d,x,a,s,so,an)|Engineers:A(r,w,rd,wd,d,x,s,so,an)|World:A(r,rd)|Asset Analytics:A(r,w,rd,wd,x,an)|Asset Analytics Recalculation:A(x)|Notifications:A(r,w,rd,wd,an)|PIVisionService:A(r,w,rd,wd,d,x,a,s,so,an)|RTQP Engine:A(r,rd)|TECHNOLOGY-ADMINISTRATOR:A(r,w,rd,wd,d,x,a,s,so,an)|APPLICATION-GEOTECDATAGATHERING:A(r,w,rd,wd,d,x,a,s,so,an)|GEOTECHNICS-BASICOPERATOR:A(r,rd)|GEOTECHNICS-NORMATIVE:A(r,w,rd,wd,d,x,a,s,so,an)|TECHNOLOGY-BASICOPERATOR:A(r,rd) + + + FrozenStatus + + + + + Enabled + false + High + 0 + 0 + + Company_name\Area01\Country\Location 02\Location 03\Element 01 + + Administrators:A(r,w,rd,wd,d,x,a,s,so,an)|Engineers:A(r,w,rd,wd,d,x,s,so,an)|World:A(r,rd)|Asset Analytics:A(r,w,rd,wd,x,an)|Asset Analytics Recalculation:A(x)|Notifications:A(r,w,rd,wd,an)|PIVisionService:A(r,w,rd,wd,d,x,a,s,so,an)|RTQP Engine:A(r,rd)|TECHNOLOGY-ADMINISTRATOR:A(r,w,rd,wd,d,x,a,s,so,an)|APPLICATION-GEOTECDATAGATHERING:A(r,w,rd,wd,d,x,a,s,so,an)|GEOTECHNICS-BASICOPERATOR:A(r,rd)|GEOTECHNICS-NORMATIVE:A(r,w,rd,wd,d,x,a,s,so,an)|TECHNOLOGY-BASICOPERATOR:A(r,rd) + + + HealthStatus + + + + + Enabled + false + High + 0 + 0 + + Company_name\Area01\Country\Location 02\Location 03\Element 01 + + Administrators:A(r,w,rd,wd,d,x,a,s,so,an)|Engineers:A(r,w,rd,wd,d,x,s,so,an)|World:A(r,rd)|Asset Analytics:A(r,w,rd,wd,x,an)|Asset Analytics Recalculation:A(x)|Notifications:A(r,w,rd,wd,an)|PIVisionService:A(r,w,rd,wd,d,x,a,s,so,an)|RTQP Engine:A(r,rd)|TECHNOLOGY-ADMINISTRATOR:A(r,w,rd,wd,d,x,a,s,so,an)|APPLICATION-GEOTECDATAGATHERING:A(r,w,rd,wd,d,x,a,s,so,an)|GEOTECHNICS-BASICOPERATOR:A(r,rd)|GEOTECHNICS-NORMATIVE:A(r,w,rd,wd,d,x,a,s,so,an)|TECHNOLOGY-BASICOPERATOR:A(r,rd) + + + LastActualization + + + + + Enabled + false + High + 0 + 0 + + Company_name\Area01\Country\Location 02\Location 03\Element 01 + + Administrators:A(r,w,rd,wd,d,x,a,s,so,an)|Engineers:A(r,w,rd,wd,d,x,s,so,an)|World:A(r,rd)|Asset Analytics:A(r,w,rd,wd,x,an)|Asset Analytics Recalculation:A(x)|Notifications:A(r,w,rd,wd,an)|PIVisionService:A(r,w,rd,wd,d,x,a,s,so,an)|RTQP Engine:A(r,rd)|TECHNOLOGY-ADMINISTRATOR:A(r,w,rd,wd,d,x,a,s,so,an)|APPLICATION-GEOTECDATAGATHERING:A(r,w,rd,wd,d,x,a,s,so,an)|GEOTECHNICS-BASICOPERATOR:A(r,rd)|GEOTECHNICS-NORMATIVE:A(r,w,rd,wd,d,x,a,s,so,an)|TECHNOLOGY-BASICOPERATOR:A(r,rd) + + + Tag Bad Status + + + + + Enabled + false + High + 0 + 0 + + Company_name\Area01\Country\Location 02\Location 03\Element 01 + + Administrators:A(r,w,rd,wd,d,x,a,s,so,an)|Engineers:A(r,w,rd,wd,d,x,s,so,an)|World:A(r,rd)|Asset Analytics:A(r,w,rd,wd,x,an)|Asset Analytics Recalculation:A(x)|Notifications:A(r,w,rd,wd,an)|PIVisionService:A(r,w,rd,wd,d,x,a,s,so,an)|RTQP Engine:A(r,rd)|TECHNOLOGY-ADMINISTRATOR:A(r,w,rd,wd,d,x,a,s,so,an)|APPLICATION-GEOTECDATAGATHERING:A(r,w,rd,wd,d,x,a,s,so,an)|GEOTECHNICS-BASICOPERATOR:A(r,rd)|GEOTECHNICS-NORMATIVE:A(r,w,rd,wd,d,x,a,s,so,an)|TECHNOLOGY-BASICOPERATOR:A(r,rd) + + + Value Filtered + + + + + Enabled + false + High + 0 + 0 + + Company_name\Area01\Country\Location 02\Location 03\Element 01 + + Administrators:A(r,w,rd,wd,d,x,a,s,so,an)|Engineers:A(r,w,rd,wd,d,x,s,so,an)|World:A(r,rd)|Asset Analytics:A(r,w,rd,wd,x,an)|Asset Analytics Recalculation:A(x)|Notifications:A(r,w,rd,wd,an)|PIVisionService:A(r,w,rd,wd,d,x,a,s,so,an)|RTQP Engine:A(r,rd)|TECHNOLOGY-ADMINISTRATOR:A(r,w,rd,wd,d,x,a,s,so,an)|APPLICATION-GEOTECDATAGATHERING:A(r,w,rd,wd,d,x,a,s,so,an)|GEOTECHNICS-BASICOPERATOR:A(r,rd)|GEOTECHNICS-NORMATIVE:A(r,w,rd,wd,d,x,a,s,so,an)|TECHNOLOGY-BASICOPERATOR:A(r,rd) + + + + Last Level 02 + + + false + DOMAIN\username + + 1970-01-01T00:00:00Z + 9999-12-31T23:59:59Z + + Alarme + + false + false + false + false + + + -5 + Double + + Good + 1970-01-01T00:00:00Z + 17 + + + + + Group + + false + false + false + false + + + -5 + AFEnumerationValue + EnumerationSets[Goup 01] + + + + + + Controller + + false + false + false + false + + + -5 + AFEnumerationValue + EnumerationSets[On/Off] + + + + + + Availibility Average + + false + false + false + false + + % + -5 + Double + + PI Point + \\name_of_the_pi_server?server_id\Location 03.Availibility Average?43257 + + + Count + + false + false + false + false + + + -5 + Double + + Good + 1970-01-01T00:00:00Z + 145 + + + + + Name + + false + false + false + false + + + -5 + String + + String Builder + %Element% + + + Availibility Calc + + + + + Enabled + false + High + 0 + 0 + + Company_name\Area01\Country\Location 02\Location 03\Element 01 + + Administrators:A(r,w,rd,wd,d,x,a,s,so,an)|Engineers:A(r,w,rd,wd,d,x,s,so,an)|World:A(r,rd)|Asset Analytics:A(r,w,rd,wd,x,an)|Asset Analytics Recalculation:A(x)|Notifications:A(r,w,rd,wd,an)|PIVisionService:A(r,w,rd,wd,d,x,a,s,so,an)|RTQP Engine:A(r,rd)|TECHNOLOGY-ADMINISTRATOR:A(r,w,rd,wd,d,x,a,s,so,an)|APPLICATION-GEOTECDATAGATHERING:A(r,w,rd,wd,d,x,a,s,so,an)|GEOTECHNICS-BASICOPERATOR:A(r,rd)|GEOTECHNICS-NORMATIVE:A(r,w,rd,wd,d,x,a,s,so,an)|TECHNOLOGY-BASICOPERATOR:A(r,rd) + + + FrozenStatus + + + + + Enabled + false + High + 0 + 0 + + Company_name\Area01\Country\Location 02\Location 03\Element 01 + + Administrators:A(r,w,rd,wd,d,x,a,s,so,an)|Engineers:A(r,w,rd,wd,d,x,s,so,an)|World:A(r,rd)|Asset Analytics:A(r,w,rd,wd,x,an)|Asset Analytics Recalculation:A(x)|Notifications:A(r,w,rd,wd,an)|PIVisionService:A(r,w,rd,wd,d,x,a,s,so,an)|RTQP Engine:A(r,rd)|TECHNOLOGY-ADMINISTRATOR:A(r,w,rd,wd,d,x,a,s,so,an)|APPLICATION-GEOTECDATAGATHERING:A(r,w,rd,wd,d,x,a,s,so,an)|GEOTECHNICS-BASICOPERATOR:A(r,rd)|GEOTECHNICS-NORMATIVE:A(r,w,rd,wd,d,x,a,s,so,an)|TECHNOLOGY-BASICOPERATOR:A(r,rd) + + + HealthStatus + + + + + Enabled + false + High + 0 + 0 + + Company_name\Area01\Country\Location 02\Location 03\Element 01 + + Administrators:A(r,w,rd,wd,d,x,a,s,so,an)|Engineers:A(r,w,rd,wd,d,x,s,so,an)|World:A(r,rd)|Asset Analytics:A(r,w,rd,wd,x,an)|Asset Analytics Recalculation:A(x)|Notifications:A(r,w,rd,wd,an)|PIVisionService:A(r,w,rd,wd,d,x,a,s,so,an)|RTQP Engine:A(r,rd)|TECHNOLOGY-ADMINISTRATOR:A(r,w,rd,wd,d,x,a,s,so,an)|APPLICATION-GEOTECDATAGATHERING:A(r,w,rd,wd,d,x,a,s,so,an)|GEOTECHNICS-BASICOPERATOR:A(r,rd)|GEOTECHNICS-NORMATIVE:A(r,w,rd,wd,d,x,a,s,so,an)|TECHNOLOGY-BASICOPERATOR:A(r,rd) + + + LastActualization + + + + + Enabled + false + High + 0 + 0 + + Company_name\Area01\Country\Location 02\Location 03\Element 01 + + Administrators:A(r,w,rd,wd,d,x,a,s,so,an)|Engineers:A(r,w,rd,wd,d,x,s,so,an)|World:A(r,rd)|Asset Analytics:A(r,w,rd,wd,x,an)|Asset Analytics Recalculation:A(x)|Notifications:A(r,w,rd,wd,an)|PIVisionService:A(r,w,rd,wd,d,x,a,s,so,an)|RTQP Engine:A(r,rd)|TECHNOLOGY-ADMINISTRATOR:A(r,w,rd,wd,d,x,a,s,so,an)|APPLICATION-GEOTECDATAGATHERING:A(r,w,rd,wd,d,x,a,s,so,an)|GEOTECHNICS-BASICOPERATOR:A(r,rd)|GEOTECHNICS-NORMATIVE:A(r,w,rd,wd,d,x,a,s,so,an)|TECHNOLOGY-BASICOPERATOR:A(r,rd) + + + Tag Bad Status + + + + + Enabled + false + High + 0 + 0 + + Company_name\Area01\Country\Location 02\Location 03\Element 01 + + Administrators:A(r,w,rd,wd,d,x,a,s,so,an)|Engineers:A(r,w,rd,wd,d,x,s,so,an)|World:A(r,rd)|Asset Analytics:A(r,w,rd,wd,x,an)|Asset Analytics Recalculation:A(x)|Notifications:A(r,w,rd,wd,an)|PIVisionService:A(r,w,rd,wd,d,x,a,s,so,an)|RTQP Engine:A(r,rd)|TECHNOLOGY-ADMINISTRATOR:A(r,w,rd,wd,d,x,a,s,so,an)|APPLICATION-GEOTECDATAGATHERING:A(r,w,rd,wd,d,x,a,s,so,an)|GEOTECHNICS-BASICOPERATOR:A(r,rd)|GEOTECHNICS-NORMATIVE:A(r,w,rd,wd,d,x,a,s,so,an)|TECHNOLOGY-BASICOPERATOR:A(r,rd) + + + Value Filtered + + + + + Enabled + false + High + 0 + 0 + + Company_name\Area01\Country\Location 02\Location 03\Element 01 + + Administrators:A(r,w,rd,wd,d,x,a,s,so,an)|Engineers:A(r,w,rd,wd,d,x,s,so,an)|World:A(r,rd)|Asset Analytics:A(r,w,rd,wd,x,an)|Asset Analytics Recalculation:A(x)|Notifications:A(r,w,rd,wd,an)|PIVisionService:A(r,w,rd,wd,d,x,a,s,so,an)|RTQP Engine:A(r,rd)|TECHNOLOGY-ADMINISTRATOR:A(r,w,rd,wd,d,x,a,s,so,an)|APPLICATION-GEOTECDATAGATHERING:A(r,w,rd,wd,d,x,a,s,so,an)|GEOTECHNICS-BASICOPERATOR:A(r,rd)|GEOTECHNICS-NORMATIVE:A(r,w,rd,wd,d,x,a,s,so,an)|TECHNOLOGY-BASICOPERATOR:A(r,rd) + + + + Last Level 03 + + + false + DOMAIN\username + + 1970-01-01T00:00:00Z + 9999-12-31T23:59:59Z + + Alarme + + false + false + false + false + + + -5 + Double + + Good + 1970-01-01T00:00:00Z + 17 + + + + + Group + + false + false + false + false + + + -5 + AFEnumerationValue + EnumerationSets[Goup 01] + + + + + + Controller + + false + false + false + false + + + -5 + AFEnumerationValue + EnumerationSets[On/Off] + + + + + + Availibility Average + + false + false + false + false + + % + -5 + Double + + PI Point + \\name_of_the_pi_server?server_id\Location 03.Availibility Average?43257 + + + Count + + false + false + false + false + + + -5 + Double + + Good + 1970-01-01T00:00:00Z + 145 + + + + + Name + + false + false + false + false + + + -5 + String + + String Builder + %Element% + + + Availibility Calc + + + + + Enabled + false + High + 0 + 0 + + Company_name\Area01\Country\Location 02\Location 03\Element 01 + + Administrators:A(r,w,rd,wd,d,x,a,s,so,an)|Engineers:A(r,w,rd,wd,d,x,s,so,an)|World:A(r,rd)|Asset Analytics:A(r,w,rd,wd,x,an)|Asset Analytics Recalculation:A(x)|Notifications:A(r,w,rd,wd,an)|PIVisionService:A(r,w,rd,wd,d,x,a,s,so,an)|RTQP Engine:A(r,rd)|TECHNOLOGY-ADMINISTRATOR:A(r,w,rd,wd,d,x,a,s,so,an)|APPLICATION-GEOTECDATAGATHERING:A(r,w,rd,wd,d,x,a,s,so,an)|GEOTECHNICS-BASICOPERATOR:A(r,rd)|GEOTECHNICS-NORMATIVE:A(r,w,rd,wd,d,x,a,s,so,an)|TECHNOLOGY-BASICOPERATOR:A(r,rd) + + + FrozenStatus + + + + + Enabled + false + High + 0 + 0 + + Company_name\Area01\Country\Location 02\Location 03\Element 01 + + Administrators:A(r,w,rd,wd,d,x,a,s,so,an)|Engineers:A(r,w,rd,wd,d,x,s,so,an)|World:A(r,rd)|Asset Analytics:A(r,w,rd,wd,x,an)|Asset Analytics Recalculation:A(x)|Notifications:A(r,w,rd,wd,an)|PIVisionService:A(r,w,rd,wd,d,x,a,s,so,an)|RTQP Engine:A(r,rd)|TECHNOLOGY-ADMINISTRATOR:A(r,w,rd,wd,d,x,a,s,so,an)|APPLICATION-GEOTECDATAGATHERING:A(r,w,rd,wd,d,x,a,s,so,an)|GEOTECHNICS-BASICOPERATOR:A(r,rd)|GEOTECHNICS-NORMATIVE:A(r,w,rd,wd,d,x,a,s,so,an)|TECHNOLOGY-BASICOPERATOR:A(r,rd) + + + HealthStatus + + + + + Enabled + false + High + 0 + 0 + + Company_name\Area01\Country\Location 02\Location 03\Element 01 + + Administrators:A(r,w,rd,wd,d,x,a,s,so,an)|Engineers:A(r,w,rd,wd,d,x,s,so,an)|World:A(r,rd)|Asset Analytics:A(r,w,rd,wd,x,an)|Asset Analytics Recalculation:A(x)|Notifications:A(r,w,rd,wd,an)|PIVisionService:A(r,w,rd,wd,d,x,a,s,so,an)|RTQP Engine:A(r,rd)|TECHNOLOGY-ADMINISTRATOR:A(r,w,rd,wd,d,x,a,s,so,an)|APPLICATION-GEOTECDATAGATHERING:A(r,w,rd,wd,d,x,a,s,so,an)|GEOTECHNICS-BASICOPERATOR:A(r,rd)|GEOTECHNICS-NORMATIVE:A(r,w,rd,wd,d,x,a,s,so,an)|TECHNOLOGY-BASICOPERATOR:A(r,rd) + + + LastActualization + + + + + Enabled + false + High + 0 + 0 + + Company_name\Area01\Country\Location 02\Location 03\Element 01 + + Administrators:A(r,w,rd,wd,d,x,a,s,so,an)|Engineers:A(r,w,rd,wd,d,x,s,so,an)|World:A(r,rd)|Asset Analytics:A(r,w,rd,wd,x,an)|Asset Analytics Recalculation:A(x)|Notifications:A(r,w,rd,wd,an)|PIVisionService:A(r,w,rd,wd,d,x,a,s,so,an)|RTQP Engine:A(r,rd)|TECHNOLOGY-ADMINISTRATOR:A(r,w,rd,wd,d,x,a,s,so,an)|APPLICATION-GEOTECDATAGATHERING:A(r,w,rd,wd,d,x,a,s,so,an)|GEOTECHNICS-BASICOPERATOR:A(r,rd)|GEOTECHNICS-NORMATIVE:A(r,w,rd,wd,d,x,a,s,so,an)|TECHNOLOGY-BASICOPERATOR:A(r,rd) + + + Tag Bad Status + + + + + Enabled + false + High + 0 + 0 + + Company_name\Area01\Country\Location 02\Location 03\Element 01 + + Administrators:A(r,w,rd,wd,d,x,a,s,so,an)|Engineers:A(r,w,rd,wd,d,x,s,so,an)|World:A(r,rd)|Asset Analytics:A(r,w,rd,wd,x,an)|Asset Analytics Recalculation:A(x)|Notifications:A(r,w,rd,wd,an)|PIVisionService:A(r,w,rd,wd,d,x,a,s,so,an)|RTQP Engine:A(r,rd)|TECHNOLOGY-ADMINISTRATOR:A(r,w,rd,wd,d,x,a,s,so,an)|APPLICATION-GEOTECDATAGATHERING:A(r,w,rd,wd,d,x,a,s,so,an)|GEOTECHNICS-BASICOPERATOR:A(r,rd)|GEOTECHNICS-NORMATIVE:A(r,w,rd,wd,d,x,a,s,so,an)|TECHNOLOGY-BASICOPERATOR:A(r,rd) + + + Value Filtered + + + + + Enabled + false + High + 0 + 0 + + Company_name\Area01\Country\Location 02\Location 03\Element 01 + + Administrators:A(r,w,rd,wd,d,x,a,s,so,an)|Engineers:A(r,w,rd,wd,d,x,s,so,an)|World:A(r,rd)|Asset Analytics:A(r,w,rd,wd,x,an)|Asset Analytics Recalculation:A(x)|Notifications:A(r,w,rd,wd,an)|PIVisionService:A(r,w,rd,wd,d,x,a,s,so,an)|RTQP Engine:A(r,rd)|TECHNOLOGY-ADMINISTRATOR:A(r,w,rd,wd,d,x,a,s,so,an)|APPLICATION-GEOTECDATAGATHERING:A(r,w,rd,wd,d,x,a,s,so,an)|GEOTECHNICS-BASICOPERATOR:A(r,rd)|GEOTECHNICS-NORMATIVE:A(r,w,rd,wd,d,x,a,s,so,an)|TECHNOLOGY-BASICOPERATOR:A(r,rd) + + + + + + + + + \ No newline at end of file diff --git a/examples/tag_db/meta_model/tag_db_type_def.json b/examples/tag_db/meta_model/tag_db_type_def.json new file mode 100644 index 0000000..3587bbd --- /dev/null +++ b/examples/tag_db/meta_model/tag_db_type_def.json @@ -0,0 +1,649 @@ +[ + { + "category": "ENTITY", + "name": "afdatabase", + "description": "AFDatabase", + "serviceType": "osi_pi", + "options": {}, + "attributeDefs": [ + { + "name": "DefaultPIServer", + "typeName": "string", + "isOptional": false, + "cardinality": "SINGLE", + "valuesMinCount": 1, + "valuesMaxCount": 1, + "isUnique": true, + "isIndexable": false, + "includeInNotification": false + }, + { + "name": "DefaultPIServerID", + "typeName": "string", + "isOptional": false, + "cardinality": "SINGLE", + "valuesMinCount": 1, + "valuesMaxCount": 1, + "isUnique": true, + "isIndexable": false, + "includeInNotification": false + } + ], + "superTypes": [ + "Asset" + ], + "subTypes": [], + "relationshipAttributeDefs": [ + { + "name": "Database", + "typeName": "afelement", + "isOptional": false, + "cardinality": "SINGLE", + "valuesMinCount": -1, + "valuesMaxCount": -1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "relationshipTypeName": "afdatabase_afelement", + "isLegacyAttribute": false + }, + { + "name": "meanings", + "typeName": "array", + "isOptional": true, + "cardinality": "SET", + "valuesMinCount": -1, + "valuesMaxCount": -1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "relationshipTypeName": "AtlasGlossarySemanticAssignment", + "isLegacyAttribute": false + } + ] + }, + { + "category": "ENTITY", + "name": "afelement", + "description": "AFElement", + "serviceType": "osi_pi", + "options": {}, + "attributeDefs": [ + { + "name": "Template", + "typeName": "string", + "isOptional": false, + "cardinality": "SINGLE", + "valuesMinCount": 1, + "valuesMaxCount": 1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false + }, + { + "name": "IsAnnotated", + "typeName": "int", + "isOptional": false, + "cardinality": "SINGLE", + "valuesMinCount": 1, + "valuesMaxCount": 1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false + }, + { + "name": "Modifier", + "typeName": "string", + "isOptional": false, + "cardinality": "SINGLE", + "valuesMinCount": 1, + "valuesMaxCount": 1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false + }, + { + "name": "Comment", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "valuesMinCount": 0, + "valuesMaxCount": 1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false + }, + { + "name": "EffectiveDate", + "typeName": "date", + "isOptional": false, + "cardinality": "SINGLE", + "valuesMinCount": 1, + "valuesMaxCount": 1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false + }, + { + "name": "ObsoleteDate", + "typeName": "date", + "isOptional": false, + "cardinality": "SINGLE", + "valuesMinCount": 1, + "valuesMaxCount": 1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false + } + ], + "superTypes": [ + "Asset" + ], + "subTypes": [], + "relationshipAttributeDefs": [ + { + "name": "Parent", + "typeName": "afelement", + "isOptional": false, + "cardinality": "SINGLE", + "valuesMinCount": -1, + "valuesMaxCount": -1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "relationshipTypeName": "afelement_afelement", + "isLegacyAttribute": false + }, + { + "name": "Element", + "typeName": "afattribute", + "isOptional": false, + "cardinality": "SINGLE", + "valuesMinCount": -1, + "valuesMaxCount": -1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "relationshipTypeName": "afelement_afattribute", + "isLegacyAttribute": false + }, + { + "name": "Element", + "typeName": "array", + "isOptional": true, + "cardinality": "SET", + "valuesMinCount": -1, + "valuesMaxCount": -1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "constraints": [ + { + "type": "ownedRef" + } + ], + "relationshipTypeName": "afdatabase_afelement", + "isLegacyAttribute": false + }, + { + "name": "Element", + "typeName": "afanalysis", + "isOptional": false, + "cardinality": "SINGLE", + "valuesMinCount": -1, + "valuesMaxCount": -1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "relationshipTypeName": "afelement_afanalysis", + "isLegacyAttribute": false + }, + { + "name": "Child", + "typeName": "array", + "isOptional": true, + "cardinality": "SET", + "valuesMinCount": -1, + "valuesMaxCount": -1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "constraints": [ + { + "type": "ownedRef" + } + ], + "relationshipTypeName": "afelement_afelement", + "isLegacyAttribute": false + }, + { + "name": "meanings", + "typeName": "array", + "isOptional": true, + "cardinality": "SET", + "valuesMinCount": -1, + "valuesMaxCount": -1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "relationshipTypeName": "AtlasGlossarySemanticAssignment", + "isLegacyAttribute": false + } + ] + }, + { + "category": "ENTITY", + "name": "afattribute", + "description": "AFAttribute", + "serviceType": "osi_pi", + "options": {}, + "attributeDefs": [ + { + "name": "IsHidden", + "typeName": "int", + "isOptional": false, + "cardinality": "SINGLE", + "valuesMinCount": 1, + "valuesMaxCount": 1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false + }, + { + "name": "IsManualDataEntry", + "typeName": "int", + "isOptional": false, + "cardinality": "SINGLE", + "valuesMinCount": 1, + "valuesMaxCount": 1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false + }, + { + "name": "IsConfigurationItem", + "typeName": "int", + "isOptional": false, + "cardinality": "SINGLE", + "valuesMinCount": 1, + "valuesMaxCount": 1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false + }, + { + "name": "IsExcluded", + "typeName": "int", + "isOptional": false, + "cardinality": "SINGLE", + "valuesMinCount": 1, + "valuesMaxCount": 1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false + }, + { + "name": "Trait", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "valuesMinCount": 0, + "valuesMaxCount": 1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false + }, + { + "name": "DefaultUOM", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "valuesMinCount": 0, + "valuesMaxCount": 1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false + }, + { + "name": "DisplayDigits", + "typeName": "int", + "isOptional": false, + "cardinality": "SINGLE", + "valuesMinCount": 1, + "valuesMaxCount": 1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false + }, + { + "name": "Type", + "typeName": "string", + "isOptional": false, + "cardinality": "SINGLE", + "valuesMinCount": 1, + "valuesMaxCount": 1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false + }, + { + "name": "TypeQualifier", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "valuesMinCount": 0, + "valuesMaxCount": 1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false + }, + { + "name": "DataReference", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "valuesMinCount": 0, + "valuesMaxCount": 1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false + }, + { + "name": "ConfigString", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "valuesMinCount": 0, + "valuesMaxCount": 1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false + }, + { + "name": "Value", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "valuesMinCount": 0, + "valuesMaxCount": 1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false + }, + { + "name": "AFAttributeCategoryRef", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "valuesMinCount": 0, + "valuesMaxCount": 1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false + } + ], + "superTypes": [ + "Asset" + ], + "subTypes": [], + "relationshipAttributeDefs": [ + { + "name": "Attribute", + "typeName": "array", + "isOptional": true, + "cardinality": "SET", + "valuesMinCount": -1, + "valuesMaxCount": -1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "constraints": [ + { + "type": "ownedRef" + } + ], + "relationshipTypeName": "afelement_afattribute", + "isLegacyAttribute": false + }, + { + "name": "meanings", + "typeName": "array", + "isOptional": true, + "cardinality": "SET", + "valuesMinCount": -1, + "valuesMaxCount": -1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "relationshipTypeName": "AtlasGlossarySemanticAssignment", + "isLegacyAttribute": false + } + ] + }, + { + "category": "ENTITY", + "name": "afanalysis", + "description": "AFAnalysis", + "serviceType": "osi_pi", + "options": {}, + "attributeDefs": [ + { + "name": "Template", + "typeName": "string", + "isOptional": false, + "cardinality": "SINGLE", + "valuesMinCount": 1, + "valuesMaxCount": 1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false + }, + { + "name": "CaseTemplate", + "typeName": "string", + "isOptional": false, + "cardinality": "SINGLE", + "valuesMinCount": 1, + "valuesMaxCount": 1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false + }, + { + "name": "OutputTime", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "valuesMinCount": 0, + "valuesMaxCount": 1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false + }, + { + "name": "Status", + "typeName": "string", + "isOptional": false, + "cardinality": "SINGLE", + "valuesMinCount": 1, + "valuesMaxCount": 1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false + }, + { + "name": "PublishResults", + "typeName": "int", + "isOptional": false, + "cardinality": "SINGLE", + "valuesMinCount": 1, + "valuesMaxCount": 1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false + }, + { + "name": "Priority", + "typeName": "string", + "isOptional": false, + "cardinality": "SINGLE", + "valuesMinCount": 1, + "valuesMaxCount": 1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false + }, + { + "name": "MaxQueueSize", + "typeName": "int", + "isOptional": false, + "cardinality": "SINGLE", + "valuesMinCount": 1, + "valuesMaxCount": 1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false + }, + { + "name": "GroupID", + "typeName": "int", + "isOptional": false, + "cardinality": "SINGLE", + "valuesMinCount": 1, + "valuesMaxCount": 1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false + } + ], + "superTypes": [ + "Asset" + ], + "subTypes": [], + "relationshipAttributeDefs": [ + { + "name": "Analysis", + "typeName": "array", + "isOptional": true, + "cardinality": "SET", + "valuesMinCount": -1, + "valuesMaxCount": -1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "constraints": [ + { + "type": "ownedRef" + } + ], + "relationshipTypeName": "afelement_afanalysis", + "isLegacyAttribute": false + }, + { + "name": "meanings", + "typeName": "array", + "isOptional": true, + "cardinality": "SET", + "valuesMinCount": -1, + "valuesMaxCount": -1, + "isUnique": false, + "isIndexable": false, + "includeInNotification": false, + "relationshipTypeName": "AtlasGlossarySemanticAssignment", + "isLegacyAttribute": false + } + ] + }, + { + "category": "RELATIONSHIP", + "name": "afdatabase_afelement", + "description": "AFDatabase_AFElement", + "serviceType": "osi_pi", + "attributeDefs": [], + "relationshipCategory": "COMPOSITION", + "propagateTags": "NONE", + "endDef1": { + "type": "afdatabase", + "name": "Element", + "isContainer": true, + "cardinality": "SET", + "isLegacyAttribute": false + }, + "endDef2": { + "type": "afelement", + "name": "Database", + "isContainer": false, + "cardinality": "SINGLE", + "isLegacyAttribute": false + } + }, + { + "category": "RELATIONSHIP", + "name": "afelement_afelement", + "description": "AFElement_AFElement", + "serviceType": "osi_pi", + "attributeDefs": [], + "relationshipCategory": "COMPOSITION", + "propagateTags": "NONE", + "endDef1": { + "type": "afelement", + "name": "Child", + "isContainer": true, + "cardinality": "SET", + "isLegacyAttribute": false + }, + "endDef2": { + "type": "afelement", + "name": "Parent", + "isContainer": false, + "cardinality": "SINGLE", + "isLegacyAttribute": false + } + }, + { + "category": "RELATIONSHIP", + "name": "afelement_afattribute", + "description": "AFElement_AFAttribute", + "serviceType": "osi_pi", + "attributeDefs": [], + "relationshipCategory": "COMPOSITION", + "propagateTags": "NONE", + "endDef2": { + "type": "afattribute", + "name": "Parent Element", + "isContainer": false, + "cardinality": "SINGLE", + "isLegacyAttribute": false + }, + "endDef1": { + "type": "afelement", + "name": "Attribute", + "isContainer": true, + "cardinality": "SET", + "isLegacyAttribute": false + } + }, + { + "category": "RELATIONSHIP", + "name": "afelement_afanalysis", + "description": "AFElement_AFAnalysis", + "serviceType": "osi_pi", + "attributeDefs": [], + "relationshipCategory": "COMPOSITION", + "propagateTags": "NONE", + "endDef2": { + "type": "afanalysis", + "name": "Reference Element", + "isContainer": false, + "cardinality": "SINGLE", + "isLegacyAttribute": false + }, + "endDef1": { + "type": "afelement", + "name": "Analysis", + "isContainer": true, + "cardinality": "SET", + "isLegacyAttribute": false + } + } +] \ No newline at end of file diff --git a/examples/tag_db/synapse/dataset/TAG_DB_JSON.json b/examples/tag_db/synapse/dataset/TAG_DB_JSON.json new file mode 100644 index 0000000..0257417 --- /dev/null +++ b/examples/tag_db/synapse/dataset/TAG_DB_JSON.json @@ -0,0 +1,23 @@ +{ + "name": "TAG_DB_JSON", + "properties": { + "linkedServiceName": { + "referenceName": "purviewaccws-WorkspaceDefaultStorage", + "type": "LinkedServiceReference" + }, + "annotations": [], + "type": "Json", + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "fileName": { + "value": "@concat(concat('tag_db',string(ticks(utcnow()))),'.json')", + "type": "Expression" + }, + "folderPath": "pccsa_main/tag-db-connector/tag-db-json", + "fileSystem": "pccsa" + } + }, + "schema": {} + } +} \ No newline at end of file diff --git a/examples/tag_db/synapse/dataset/TAG_DB_Metadata_XML.json b/examples/tag_db/synapse/dataset/TAG_DB_Metadata_XML.json new file mode 100644 index 0000000..bcc3340 --- /dev/null +++ b/examples/tag_db/synapse/dataset/TAG_DB_Metadata_XML.json @@ -0,0 +1,19 @@ +{ + "name": "TAG_DB_Metadata_XML", + "properties": { + "linkedServiceName": { + "referenceName": "purviewaccws-WorkspaceDefaultStorage", + "type": "LinkedServiceReference" + }, + "annotations": [], + "type": "Xml", + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "fileName": "tag-db-xml-sample.xml", + "folderPath": "tag-db-xml", + "fileSystem": "pccsa" + } + } + } +} \ No newline at end of file diff --git a/examples/tag_db/synapse/notebook/Purview_TAG_DB_Scan.ipynb b/examples/tag_db/synapse/notebook/Purview_TAG_DB_Scan.ipynb new file mode 100644 index 0000000..714a98c --- /dev/null +++ b/examples/tag_db/synapse/notebook/Purview_TAG_DB_Scan.ipynb @@ -0,0 +1,925 @@ +{ + "nbformat": 4, + "nbformat_minor": 2, + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "metadata": { + "tags": [ + "parameters" + ] + }, + "source": [ + "blob_container_name = \"\"\r\n", + "blob_account_name = \"\"\r\n", + "blob_relative_path = \"\"\r\n", + "app_name = \"\"\r\n", + "blob_processed=\"\"\r\n", + "out_file=\"\"" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Import All dependencies Libraries\r\n", + "Make sure you import either to the cluster as workspace level or as sessionm the PyApacheAtlas packages. All the others are native." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + "import json\r\n", + "import os\r\n", + "# PyApacheAtlas packages\r\n", + "# for using guid generator to garantee unid guids\r\n", + "from pyapacheatlas.core.util import GuidTracker\r\n", + "from notebookutils import mssparkutils\r\n", + "from pyspark.conf import SparkConf\r\n", + "from pyspark.sql import SparkSession\r\n", + "from datetime import datetime" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Setting up program variables\r\n", + " - Logger: Use to logg debuging information\r\n", + " - mylogger: Object use to log\r\n", + " - adls_home: Used for the relative path for the files used by the Notebook\r\n", + " - adls_processed: Folder where processed files are putted after Notebook finishe processing the file\r\n", + " - adls_out_home: Folder where the output json used to load on purview is generated\r\n", + " - gt: Object responsible to track the unique identities for the Json objects to load onto Purview\r\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + "#Setting up variable for loging.\r\n", + "my_jars = os.environ.get(\"SPARK_HOME\")\r\n", + "myconf = SparkConf()\r\n", + "myconf.set(\"spark.jars\",\"%s/jars/log4j-1.2.17.jar\" % my_jars)\r\n", + "spark = SparkSession\\\r\n", + " .builder\\\r\n", + " .appName(\"DB2_Test\")\\\r\n", + " .config(conf = myconf) \\\r\n", + " .getOrCreate()\r\n", + "\r\n", + "Logger= spark._jvm.org.apache.log4j.Logger\r\n", + "mylogger = Logger.getLogger(app_name)\r\n", + "#file path inicializer\r\n", + "adls_home = 'abfss://%s@%s.dfs.core.windows.net/%s' % (blob_container_name, blob_account_name, blob_relative_path)\r\n", + "adls_processed = 'abfss://%s@%s.dfs.core.windows.net/%s' % (blob_container_name, blob_account_name, blob_processed)\r\n", + "adls_out_home = 'abfss://%s@%s.dfs.core.windows.net/%s' % (blob_container_name, blob_account_name, out_file)\r\n", + "#inicialize guid tracker to garantee unique guids for the purview objects\r\n", + "gt = GuidTracker()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Log Function\r\n", + "Function write booth on the spark Server logs and output on the nootebook for debug\r\n", + " - msg_type: Type of mssage to write (ERROR or INFO)\r\n", + " - msg: Message to be logged\r\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + "#function to simplify loging on files and on screen.\r\n", + "def log_msgs(msg_type,msg):\r\n", + " \r\n", + " if msg_type.upper() == \"ERROR\":\r\n", + " print('ERROR: %s' % msg)\r\n", + " mylogger.error(msg)\r\n", + " else:\r\n", + " print('INFO: %s' % msg)\r\n", + " mylogger.info(\"Fim\")\r\n", + " " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## All TAG DB Classes Definitions\r\n", + "Used to read from XML and transform into the Json to be loaded into Purview\r\n", + " - AFDatabase\r\n", + " - AFElement\r\n", + " - AFAttribute\r\n", + " - AFAnalysis" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + "class AFBaseObject:\r\n", + " \"\"\"\r\n", + " Base Class fot the OSI PI metadata Objects,\r\n", + " provide a methond that is common for all other classes\r\n", + " \"\"\"\r\n", + " def __init__(self):\r\n", + " self.attributes= {'attributes':{}}\r\n", + "\r\n", + " \"\"\"\r\n", + " Generically creates a relationship with another data type\r\n", + "\r\n", + " :param str nameElement:\r\n", + " Name that the element will have to the current Data Asset\r\n", + " e.g. \"Database\", \"Parent\", \"Child\", \"Group\"...\r\n", + "\r\n", + " :param str typeElement:\r\n", + " Name of the type fo the element that it is creating the relationship\r\n", + "\r\n", + " :param str idElement:\r\n", + " Guid fo the object that is creating the relationship to.\r\n", + "\r\n", + " :param str relationShipType::\r\n", + " Name of the type of the relationship being created\r\n", + " \"\"\"\r\n", + " def addRelationship(self,nameElement, typeElement, idElement,relationShipType):\r\n", + " self.attributes['relationshipAttributes'][nameElement]={}\r\n", + " self.attributes['relationshipAttributes'][nameElement]['guid']=idElement\r\n", + " self.attributes['relationshipAttributes'][nameElement]['typeName']= typeElement\r\n", + " self.attributes['relationshipAttributes'][nameElement]['entityStatus']= \"ACTIVE\"\r\n", + " self.attributes['relationshipAttributes'][nameElement]['displayText']= nameElement\r\n", + " self.attributes['relationshipAttributes'][nameElement]['relationshipType']= relationShipType\r\n", + " self.attributes['relationshipAttributes'][nameElement]['relationshipStatus']= \"ACTIVE\"\r\n", + " self.attributes['relationshipAttributes'][nameElement]['relationshipAttributes']={}\r\n", + " self.attributes['relationshipAttributes'][nameElement]['relationshipAttributes']['typeName']=relationShipType\r\n", + "\r\n", + " \"\"\"\r\n", + " Return the class attributes as a dictionary (json like) to be \r\n", + " consumed by any API\r\n", + " \"\"\"\r\n", + " def toJson(self):\r\n", + " return self.attributes\r\n", + " \r\n", + " def fixDate(self, date):\r\n", + " if date is None:\r\n", + " return ''\r\n", + " else:\r\n", + " return date.replace('Z','.0000000Z')\r\n", + " \r\n", + " def removeNulls(self, field, isnumber=False):\r\n", + " if field is None:\r\n", + " if isnumber:\r\n", + " return 0\r\n", + " else:\r\n", + " return ''\r\n", + " else:\r\n", + " if field == '':\r\n", + " if isnumber:\r\n", + " return 0\r\n", + " else:\r\n", + " return ''\r\n", + " return field\r\n", + " \r\n", + "class AFDatabase(AFBaseObject):\r\n", + " \"\"\"\r\n", + " AFDatabase Data Asset Type Definition, will hold all the metadata information for\r\n", + " a OSI PI AFDatabase data asset\r\n", + " \"\"\"\r\n", + "\r\n", + "\r\n", + " \"\"\"\r\n", + " Inicialize the class with all the attributes needed\r\n", + "\r\n", + " :param str name:\r\n", + " Name that will be using for the data asset\r\n", + "\r\n", + " :param str description:\r\n", + " description of the data asset\r\n", + "\r\n", + " :param str defaultpiserver:\r\n", + " Name of the Default OSI PI Server\r\n", + "\r\n", + " :param str defaultpiserverid:\r\n", + " ID of the sefault OSI PI server\r\n", + "\r\n", + " :param str guid:\r\n", + " Unique identifier of the Data Asset.\r\n", + " \"\"\"\r\n", + " def __init__(self,name=None, description=None,defaultpiserver=None, defaultpiserverid=None,guid=None):\r\n", + " self.name = self.removeNulls(name)\r\n", + " self.description= self.removeNulls(description)\r\n", + " self.defaultpiserver = self.removeNulls(defaultpiserver)\r\n", + " self.defaultpiserverid = self.removeNulls(defaultpiserverid)\r\n", + " #unique Identifier build based on the hierarchical patr AFDatabase-Name\r\n", + " self.qualifiedName = 'osipi://%s/%s' % (self.defaultpiserver,self.name)\r\n", + " self.guid = guid\r\n", + " self.typeName= 'afdatabase'\r\n", + " self.attributes = {'attributes':{}}\r\n", + " self.attributes['attributes']['name']=self.name\r\n", + " self.attributes['guid']=guid\r\n", + " self.attributes['attributes']['description']=self.description\r\n", + " self.attributes['attributes']['DefaultPIServer']=self.defaultpiserver\r\n", + " self.attributes['attributes']['DefaultPIServerID']=self.defaultpiserverid\r\n", + " self.attributes['attributes']['qualifiedName'] = self.qualifiedName\r\n", + " self.attributes['typeName']= 'afdatabase'\r\n", + " self.attributes['relationshipAttributes'] = {}\r\n", + " \r\n", + "\r\n", + "class AFElemento(AFBaseObject):\r\n", + " \"\"\"\r\n", + " AFElement Data Asset Type Definition, will hold all the metadata information for\r\n", + " a OSI PI AFElement data asset, has a relationship with AFDatabase and can have childs as: \r\n", + " * AFElements ('Parent')\r\n", + " * AFAttribiute ('Attribute')\r\n", + " * AFAnalysis ('Analysis')\r\n", + " \"\"\"\r\n", + "\r\n", + "\r\n", + " \"\"\"\r\n", + " Inicialize the class with all the attributes needed\r\n", + "\r\n", + " :param str guid:\r\n", + " Unique identifier of the Data Asset.\r\n", + "\r\n", + " :param str name:\r\n", + " Name that will be using for the data asset\r\n", + "\r\n", + " :param str description:\r\n", + " description of the data asset\r\n", + "\r\n", + " :param int isAnnotated:\r\n", + " Represent 0 if false and 1 if true\r\n", + "\r\n", + " :param str template:\r\n", + " Name of the template that the AFElement use\r\n", + "\r\n", + " :param AFDatabase database:\r\n", + " Database that the element belongs to\r\n", + "\r\n", + " :param str comment:\r\n", + " Comment about AFElement\r\n", + "\r\n", + " :param datetime effectiveDate:\r\n", + " Effective data for the AFElement \r\n", + " (Dates are not coming with miliseconds needed to be fixed, make sure you system is not using miliseconds also\r\n", + " mkae the change if need it)\r\n", + "\r\n", + " :param datatime obsoleteDate:\r\n", + " Date when the AFElement gets obsolete\r\n", + " (Dates are not coming with miliseconds needed to be fixed, make sure you system is not using miliseconds also\r\n", + " mkae the change if need it)\r\n", + "\r\n", + " :param str modifier:\r\n", + " AFElement modifier\r\n", + " \"\"\"\r\n", + " def __init__(self,guid=None, name=None, description=None,isAnnotated=None, template=None,database=None, comment=None\r\n", + " , effectiveDate=None, obsoleteDate=None, modifier=None):\r\n", + " self.name = self.removeNulls(name)\r\n", + " self.description= self.removeNulls(description)\r\n", + " #unique Identifier build based on the hierarchical patr AFDatabase-AFElement Name\r\n", + " self.qualifiedName = 'osipi://%s/%s/%s' % (database.defaultpiserver,database.name,self.name)\r\n", + " self.obsoleteDate = self.fixDate(date=obsoleteDate)\r\n", + " self.template=self.removeNulls(template)\r\n", + " self.comment=self.removeNulls(comment)\r\n", + " self.database= database\r\n", + " self.isAnnotated=self.removeNulls(isAnnotated)\r\n", + " self.effectiveDate=self.fixDate(date=effectiveDate)\r\n", + " self.attributes = {'attributes':{}}\r\n", + " self.modifier = self.removeNulls(modifier)\r\n", + " self.guid=guid\r\n", + " self.attributes['relationshipAttributes'] = {}\r\n", + " \r\n", + " self.attributes['attributes']['name']=self.name\r\n", + " self.attributes['guid']=self.guid\r\n", + " self.attributes['attributes']['description']=self.description\r\n", + " self.attributes['attributes']['ObsoleteDate']=self.obsoleteDate\r\n", + " self.attributes['attributes']['Template']=self.template\r\n", + " self.attributes['attributes']['Comment'] = self.comment\r\n", + " self.attributes['attributes']['IsAnnotated'] = self.isAnnotated\r\n", + " self.attributes['attributes']['EffectiveDate'] = self.effectiveDate\r\n", + " self.attributes['attributes']['qualifiedName'] = self.qualifiedName\r\n", + " self.attributes['attributes']['Modifier']=self.modifier\r\n", + " self.attributes['typeName']= 'afelement'\r\n", + " self.addRelationship('Database', self.database.typeName, self.database.guid,'afdatabase_afelement')\r\n", + " \r\n", + "class AFAttribute(AFBaseObject):\r\n", + " \"\"\"\r\n", + " AFAttribute Data Asset Type Definition, will hold all the metadata information for\r\n", + " a OSI PI AFAttribute data asset has relationship with some other OSI PI data Assets:\r\n", + " * AFElements ('Parent Element')\r\n", + " \"\"\"\r\n", + "\r\n", + " \"\"\"\r\n", + " Inicialize the class with all the attributes needed\r\n", + "\r\n", + " :param str guid:\r\n", + " Unique identifier of the Data Asset.\r\n", + "\r\n", + " :param str name:\r\n", + " Name that will be using for the data asset\r\n", + "\r\n", + " :param str description:\r\n", + " description of the data asset\r\n", + " \r\n", + " :param int isHidden:\r\n", + " If is a hidden attribute, 0=false 1=true\r\n", + "\r\n", + " :param int isManualDataEntry:\r\n", + " If is a manual data entry attribute, 0=false 1=true\r\n", + "\r\n", + " :param in isExcluded:\r\n", + " If is excluded attribute, 0=false 1=true\r\n", + " \r\n", + " :param in isConfigurationItem:\r\n", + " If is a configuration attribute, 0=false 1=true\r\n", + " \r\n", + " :param str trait:\r\n", + " Trait\r\n", + "\r\n", + " :param str defaultUOM:\r\n", + " Default UOM\r\n", + " \r\n", + " :param str displayDigits:\r\n", + " # display digits\r\n", + "\r\n", + " :param str _type:\r\n", + " Type \r\n", + "\r\n", + " :param str typeQualifier:\r\n", + " Type quilifier\r\n", + " \r\n", + " :param str dataReference:\r\n", + " Reference date in string format\r\n", + " \r\n", + " :param str configString:\r\n", + " Configuration string \r\n", + " \r\n", + " :param AFDatabase database:\r\n", + " Parent AFDatabase\r\n", + "\r\n", + " :param AFElement afelement:\r\n", + " Parent AFElement\r\n", + " \"\"\"\r\n", + " def __init__(self,guid=None, name=None, description=None,isHidden=None, isManualDataEntry=None,isExcluded=None, isConfigurationItem=None, trait=None, defaultUOM=None, displayDigits=None,\r\n", + " _type=None, typeQualifier=None, dataReference=None, configString=None,database = None,afelement=None):\r\n", + " self.name = self.removeNulls(name)\r\n", + " self.description= self.removeNulls(description)\r\n", + " #unique Identifier build based on the hierarchical patr AFDatabase-AFElement-Name\r\n", + " self.qualifiedName = 'osipi://%s/%s/%s/%s' % (database.defaultpiserver,database.name,afelement.name,self.name)\r\n", + " self.isHidden = 0 if self.removeNulls(isHidden).upper()=='FALSE' else 1\r\n", + " self.isManualDataEntry=0 if self.removeNulls(isManualDataEntry).upper()=='FALSE' else 1\r\n", + " self.isExcluded=0 if self.removeNulls(isExcluded).upper()=='FALSE' else 1\r\n", + " self.database= database\r\n", + " self.isConfigurationItem=0 if self.removeNulls(isConfigurationItem).upper()=='FALSE' else 1\r\n", + " self.trait=self.removeNulls(trait)\r\n", + " self.defaultUOM=self.removeNulls(defaultUOM)\r\n", + " self.displayDigits=self.removeNulls(field=displayDigits,isnumber=True)\r\n", + " self._type=self.removeNulls(_type)\r\n", + " self.typeQualifier=self.removeNulls(typeQualifier)\r\n", + " self.dataReference=self.removeNulls(dataReference)\r\n", + " self.configString=self.removeNulls(configString)\r\n", + " self.attributes = {'attributes':{}}\r\n", + " self.guid=guid\r\n", + " self.attributes['relationshipAttributes'] = {}\r\n", + "\r\n", + " self.attributes['attributes']['name']=self.name\r\n", + " self.attributes['guid']=self.guid\r\n", + " self.attributes['attributes']['description']=self.description\r\n", + " self.attributes['attributes']['IsHidden'] = self.isHidden\r\n", + " self.attributes['attributes']['IsManualDataEntry'] = self.isManualDataEntry\r\n", + " self.attributes['attributes']['IsExcluded'] = self.isExcluded\r\n", + " self.attributes['attributes']['IsConfigurationItem'] = self.isConfigurationItem\r\n", + " self.attributes['attributes']['Trait'] = self.trait\r\n", + " self.attributes['attributes']['DefaultUOM'] = self.defaultUOM\r\n", + " self.attributes['attributes']['DisplayDigits'] = self.displayDigits\r\n", + " self.attributes['attributes']['Type'] = self._type\r\n", + " self.attributes['attributes']['TypeQualifier'] = self.typeQualifier\r\n", + " self.attributes['attributes']['DataReference'] = self.dataReference\r\n", + " self.attributes['attributes']['ConfigString'] = self.configString\r\n", + " self.attributes['attributes']['qualifiedName'] = self.qualifiedName\r\n", + " self.attributes['typeName']= 'afattribute'\r\n", + "\r\n", + " #adding relatioship to the Parent AFElement\r\n", + " self.addRelationship('Parent Element', 'afelement', afelement.guid,'afelement_afattribute')\r\n", + "\r\n", + "\r\n", + "class AFAnalysis(AFBaseObject):\r\n", + " \"\"\"\r\n", + " AFAnalysis Data Asset Type Definition, will hold all the metadata information for\r\n", + " a OSI PI AFAnalysis data asset has relationship with some other OSI PI data Assets:\r\n", + " * AFElements ('Reference Element')\r\n", + " \"\"\"\r\n", + "\r\n", + " \"\"\"\r\n", + " Inicialize the class with all the attributes needed\r\n", + "\r\n", + " :param str guid:\r\n", + " Unique identifier of the Data Asset.\r\n", + "\r\n", + " :param str name:\r\n", + " Name that will be using for the data asset\r\n", + "\r\n", + " :param str description:\r\n", + " description of the data asset\r\n", + " \r\n", + " :param in template:\r\n", + " Template used for the AFAnalysis\r\n", + " \r\n", + " :param str caseTemplate:\r\n", + " Case Template\r\n", + "\r\n", + " :param str outputTime:\r\n", + " Output time\r\n", + " \r\n", + " :param str status:\r\n", + " Analysis Status\r\n", + "\r\n", + " :param str publishResults:\r\n", + " Publish Results\r\n", + "\r\n", + " :param str priority:\r\n", + " Priority\r\n", + " \r\n", + " :param str maxQueueSize:\r\n", + " Max Queue Size\r\n", + " \r\n", + " :param str groupID:\r\n", + " Group ID\r\n", + " \r\n", + " :param str target:\r\n", + " Target\r\n", + "\r\n", + " :param AFDatabase database:\r\n", + " Parent AFDatabase\r\n", + "\r\n", + " :param AFElement afelement:\r\n", + " Parent AFElement\r\n", + " \"\"\"\r\n", + " def __init__(self,guid=None, name=None, description=None,template=None, caseTemplate=None,outputTime=None, status=None, publishResults=None, priority=None, maxQueueSize=None,\r\n", + " groupID=None, target=None,database = None,afelement=None):\r\n", + " self.name = self.removeNulls(name)\r\n", + " self.description= self.removeNulls(description)\r\n", + " #unique Identifier build based on the hierarchical patr AFDatabase-AFElement-Name\r\n", + " self.qualifiedName = 'osipi://%s/%s/%s/%s' % (database.defaultpiserver,database.name,afelement.name,self.name)\r\n", + " self.template = self.removeNulls(template)\r\n", + " self.caseTemplate=self.removeNulls(caseTemplate)\r\n", + " self.publishResults=0 if self.removeNulls(publishResults).upper()=='FALSE' else 1\r\n", + " self.database= database\r\n", + " self.outputTime=self.removeNulls(outputTime)\r\n", + " self.status=self.removeNulls(status)\r\n", + " self.priority=self.removeNulls(priority)\r\n", + " self.maxQueueSize=self.removeNulls(field=maxQueueSize,isnumber=True)\r\n", + " self.groupID=self.removeNulls(field=groupID, isnumber=True)\r\n", + " self.attributes = {'attributes':{}}\r\n", + " self.guid=guid\r\n", + " self.attributes['relationshipAttributes'] = {}\r\n", + "\r\n", + " self.attributes['attributes']['name']=self.name\r\n", + " self.attributes['guid']=self.guid\r\n", + " self.attributes['attributes']['description']=self.description\r\n", + " self.attributes['attributes']['Template'] = self.template\r\n", + " self.attributes['attributes']['CaseTemplate'] = self.caseTemplate\r\n", + " self.attributes['attributes']['PublishResults'] = self.publishResults\r\n", + " self.attributes['attributes']['OutputTime'] = self.outputTime\r\n", + " self.attributes['attributes']['Status'] = self.status\r\n", + " self.attributes['attributes']['Priority'] = self.priority\r\n", + " self.attributes['attributes']['MaxQueueSize'] = self.maxQueueSize\r\n", + " self.attributes['attributes']['GroupID'] = self.groupID\r\n", + " self.attributes['attributes']['qualifiedName'] = self.qualifiedName\r\n", + " self.attributes['typeName']= 'afanalysis'\r\n", + "\r\n", + " self.addRelationship('Reference Element', 'afelement', afelement.guid,'afelement_afanalysis')\r\n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Function to hlep check if Element exist in the Dictionary\r\n", + "Check if element exists and send empty if it dos not" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + "def get_element(name=None,dictionary=None):\r\n", + " if name in dictionary:\r\n", + " return dictionary[name]\r\n", + " return ''" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Recursive Function the iterate over the AFElement hierarchy\r\n", + "Loop through all AFElements adding the hierarcgy into the json to be loaded and recreated ad relationship into Purview " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + "#Function used to transverse the hierarchical tree and create the AFElements and it relationships\r\n", + "#can be use recursively\r\n", + "def get_AFElement(db=None, parent=None, element=None):\r\n", + " afelement = None\r\n", + " if 'Name' in element:\r\n", + " #condition to AFElements with parents\r\n", + " comment = None\r\n", + " if 'Comment' in element:\r\n", + " comment=get_element(name='Comment',dictionary= element)\r\n", + "\r\n", + " afelement = AFElemento(\r\n", + " guid=gt.get_guid(), \r\n", + " name= get_element(name='Name',dictionary= element), \r\n", + " description= get_element(name='Description',dictionary= element),\r\n", + " isAnnotated= 1 if get_element(name='IsAnnotated',dictionary= element)=='True' else 0, \r\n", + " template= get_element(name='Template',dictionary= element),\r\n", + " database=db, \r\n", + " comment=comment, \r\n", + " effectiveDate= get_element(name='EffectiveDate',dictionary= element), \r\n", + " obsoleteDate=get_element(name='ObsoleteDate',dictionary= element),\r\n", + " modifier = get_element(name='Modifier',dictionary= element)\r\n", + " )\r\n", + " if parent != None:\r\n", + " #adding Parent relationship\r\n", + " afelement.addRelationship(\r\n", + " nameElement='Parent',\r\n", + " typeElement='afelement', \r\n", + " idElement = parent.guid,\r\n", + " relationShipType='afelement_afelement')\r\n", + " #Add AFElement for the list of entities to be loaded into Purview\r\n", + " purview_load_entities.append(afelement.toJson())\r\n", + " #validating is AFElement has child AFElement\r\n", + " if 'AFElement' in element:\r\n", + " #if it is a list of AFElements\r\n", + " if type(element['AFElement']) is list:\r\n", + " for item in element['AFElement']:\r\n", + " if not type(item) is list:\r\n", + " try:\r\n", + " get_AFElement(db=db,parent=afelement ,element=item)\r\n", + " except:\r\n", + " #print(item.keys())\r\n", + " errors=1\r\n", + " print(item)\r\n", + " else:\r\n", + " #Only One AFElement\r\n", + " get_AFElement(db=db,parent= afelement ,element=element['AFElement'])\r\n", + " \r\n", + " get_AFAnalysis(db=db, afelement=afelement,element=element)\r\n", + "\r\n", + " get_AFAttributes(db=db, afelement=afelement,element=element)\r\n", + "\r\n", + "def get_AFAnalysis(db=None, afelement=None, element=None):\r\n", + " #Check is ther is AF Analysis\r\n", + " if 'AFAnalysis' in element:\r\n", + " #If it is more the one\r\n", + " if type(element['AFAnalysis']) is list:\r\n", + " for attrib in element['AFAnalysis']:\r\n", + " analysis = set_AFAnalysis(analisys=element,database = db,afelement=afelement)\r\n", + " #Append the AFAnalysis to the list of Data Assets to be loaded into Purview\r\n", + " purview_load_entities.append(analysis.toJson())\r\n", + " else:\r\n", + " #Only one AFAnalysis\r\n", + " attrib = element['AFAnalysis']\r\n", + " analysis = set_AFAnalysis(analisys=element,database = db,afelement=afelement)\r\n", + " #Append the AFAnalysis to the list of Data Assets to be loaded into Purview\r\n", + " purview_load_entities.append(analysis.toJson())\r\n", + "\r\n", + "def get_AFAttributes(db=None, afelement=None, element=None):\r\n", + " #checking if AFElement has AFAttributes\r\n", + " if 'AFAttribute' in element:\r\n", + " #If it is a list of AFAttributes\r\n", + " if type(element['AFAttribute']) is list:\r\n", + " for attrib in element['AFAttribute']:\r\n", + " attribute = set_AFAttribute(attrib=element, database = db,afelement=afelement)\r\n", + " #Append the AFAttribute to the list of Data Assets to be loaded into Purview\r\n", + " purview_load_entities.append(attribute.toJson())\r\n", + " else:\r\n", + " #Only one AFAttribute\r\n", + " attrib = element['AFAttribute']\r\n", + " attribute = set_AFAttribute(attrib=element, database = db,afelement=afelement)\r\n", + " #Append the AFAttribute to the list of Data Assets to be loaded into Purview\r\n", + " purview_load_entities.append(attribute.toJson())\r\n", + "\r\n", + "\r\n", + "def set_AFAttribute(attrib=None, database=None,afelement=None):\r\n", + " return AFAttribute(\r\n", + " guid=gt.get_guid(), name=get_element('Name',attrib), \r\n", + " description=get_element('Description',attrib),isHidden=get_element('IsHidden',attrib), \r\n", + " isManualDataEntry=get_element('IsManualDataEntry',attrib),isExcluded=get_element('IsExcluded',attrib), \r\n", + " isConfigurationItem=get_element('IsConfigurationItem',attrib), trait=get_element('Trait',attrib), \r\n", + " defaultUOM=get_element('DefaultUOM',attrib), displayDigits=get_element('DisplayDigits',attrib),\r\n", + " _type=get_element('Type',attrib), typeQualifier=get_element('TypeQualifier',attrib), \r\n", + " dataReference=get_element('DataReference',attrib), configString=get_element('ConfigString',attrib),\r\n", + " database = database,afelement=afelement)\r\n", + "\r\n", + "def set_AFAnalysis(analisys, database, afelement):\r\n", + " return AFAnalysis(\r\n", + " guid=gt.get_guid(), name=analisys['Name'], \r\n", + " description= get_element('Description',analisys),\r\n", + " template= get_element('Template',analisys), caseTemplate=get_element('CaseTemplate',analisys),\r\n", + " outputTime=get_element('OutputTime',analisys), status=get_element('Status',analisys), \r\n", + " publishResults=get_element('PublishResults',analisys), priority=get_element('Priority',analisys), maxQueueSize=get_element('MaxQueueSize',analisys),\r\n", + " groupID=get_element('GroupID',analisys), target=get_element('Target',analisys),\r\n", + " database = database,afelement=afelement)\r\n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Function (load_tag_db_DataAssets) that Generate the Full Data Assets to be loaded into Purview\r\n", + "Validate Top AF nodes to generate the the json objects" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + "def load_tag_db_DataAssets(j):\r\n", + " \r\n", + " for i in j:\r\n", + " json_obj = json.loads(i)\r\n", + " #print(json_obj)\r\n", + " if 'AF' in json_obj:\r\n", + " #print('Found AF')\r\n", + " if 'AFDatabase' in json_obj['AF']:\r\n", + " if 'Name' in json_obj['AF'][\"AFDatabase\"]:\r\n", + " #print('Found AFDatabase')\r\n", + " if 'Description' in json_obj['AF'][\"AFDatabase\"]:\r\n", + " DefaultPIServer=''\r\n", + " DefaultPIServerID=''\r\n", + " if 'AFExtendedProperty' in json_obj['AF'][\"AFDatabase\"]:\r\n", + " for l in json_obj['AF'][\"AFDatabase\"]['AFExtendedProperty']:\r\n", + " if 'Name' in l:\r\n", + " if l['Name']=='DefaultPIServer':\r\n", + " DefaultPIServer= l[\"Value\"]\r\n", + " if l['Name']=='DefaultPIServerID':\r\n", + " DefaultPIServerID = l[\"Value\"]\r\n", + " db = AFDatabase(guid=gt.get_guid(),\r\n", + " name=json_obj['AF'][\"AFDatabase\"]['Name'],\r\n", + " description=json_obj['AF'][\"AFDatabase\"]['Description'],\r\n", + " defaultpiserver=DefaultPIServer,\r\n", + " defaultpiserverid=DefaultPIServerID\r\n", + " )\r\n", + " #print(db.toJson()) \r\n", + " purview_load_entities.append(db.toJson())\r\n", + " if 'AFElement' in json_obj['AF'][\"AFDatabase\"]:\r\n", + " #print('Found AFElement')\r\n", + " get_AFElement(db,None,json_obj['AF'][\"AFDatabase\"]['AFElement'])\r\n", + " #print(purview_load_entities)\r\n", + " now = datetime.now() # current date and time\r\n", + " timestamp = now.strftime(\"%y%m%d%H%M%S%f\")\r\n", + " json_value = json.dumps(purview_load_entities)\r\n", + " mssparkutils.fs.put('%s/osipi-%s.json' % (adls_out_home,timestamp),json_value, True)\r\n", + " return True\r\n", + " return False" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "## Loop over all the files on the ADLS_HOME folder to generate all json objects to load into Purview\r\n", + "Loop over all files, load one by one and move to processs folder after correctly proccessed" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + "import traceback\r\n", + "try:\r\n", + " havefiles = True\r\n", + " inicialnumfiles = 0\r\n", + " while havefiles:\r\n", + " havefiles = False\r\n", + " files = mssparkutils.fs.ls(adls_home)\r\n", + " numoffiles = len(files)\r\n", + " processedfiles = 0\r\n", + " failfiles=0\r\n", + " for file in files:\r\n", + " purview_load_entities=[]\r\n", + " if file.size > 0:\r\n", + " havefiles = True\r\n", + " i=0\r\n", + " filepath = \"\"\r\n", + " fileparts = file.path.split('/')\r\n", + " for filepart in fileparts:\r\n", + " if i < len(fileparts)-1:\r\n", + " filepath+='%s/' % filepart\r\n", + " i+=1\r\n", + " \r\n", + " filepath='%s/%s' % (adls_processed,file.name)\r\n", + " load_json = False\r\n", + " readComplexJSONDF=None\r\n", + " try:\r\n", + " print(file.path)\r\n", + " readComplexJSONDF = spark.read.option(\"multiLine\",\"true\").json(file.path)\r\n", + " load_json=True\r\n", + " print('Finished Loading json')\r\n", + " except Exception as e:\r\n", + " log_msgs('ERROR','Invalid Json: %s /r %s' % file.path,e.args[0])\r\n", + "\r\n", + " if load_json:\r\n", + " print('Start Loading Json')\r\n", + " j = readComplexJSONDF.toJSON().collect()\r\n", + " log_msgs('INFO','Loading File: %s' % file.path)\r\n", + " if load_tag_db_DataAssets(j):\r\n", + " print('Finished Loading File')\r\n", + " try:\r\n", + " deletfile = mssparkutils.fs.rm(filepath)\r\n", + " \r\n", + " except:\r\n", + " log_msgs('INFO','No file to delete')\r\n", + " movefile = mssparkutils.fs.mv(src=file.path,dest=filepath)\r\n", + " processedfiles+=1\r\n", + " else:\r\n", + " failfiles+=1\r\n", + " if failfiles > 0 and processedfiles == 0:\r\n", + " print('Exit all files loaded')\r\n", + " break\r\n", + "except Exception as e:\r\n", + " traceback_lines = traceback.format_exc().splitlines()\r\n", + " log_msgs('ERROR',traceback_lines)" + ] + } + ], + "metadata": { + "save_output": true, + "kernelspec": { + "name": "synapse_pyspark", + "display_name": "Synapse PySpark" + }, + "language_info": { + "name": "python" + }, + "synapse_widget": { + "version": "0.1", + "state": {} + } + } +} \ No newline at end of file diff --git a/examples/tag_db/synapse/pipeline/Converte TAG DB XML Metadata to Json.json b/examples/tag_db/synapse/pipeline/Converte TAG DB XML Metadata to Json.json new file mode 100644 index 0000000..325eae0 --- /dev/null +++ b/examples/tag_db/synapse/pipeline/Converte TAG DB XML Metadata to Json.json @@ -0,0 +1,125 @@ +{ + "name": "Converte TAG DB XML Metadata to Json", + "properties": { + "activities": [ + { + "name": "Convert TAG DB XML METADATA to JSON", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "XmlSource", + "storeSettings": { + "type": "AzureBlobFSReadSettings", + "recursive": true, + "wildcardFolderPath": { + "value": "@pipeline().parameters.blob_xml_path", + "type": "Expression" + }, + "wildcardFileName": "tag-db-xml-sample.xml", + "enablePartitionDiscovery": false + }, + "formatSettings": { + "type": "XmlReadSettings", + "validationMode": "none", + "namespaces": true + } + }, + "sink": { + "type": "JsonSink", + "storeSettings": { + "type": "AzureBlobFSWriteSettings" + }, + "formatSettings": { + "type": "JsonWriteSettings" + } + }, + "enableStaging": false + }, + "inputs": [ + { + "referenceName": "TAG_DB_Metadata_XML", + "type": "DatasetReference" + } + ], + "outputs": [ + { + "referenceName": "TAG_DB_JSON", + "type": "DatasetReference" + } + ] + }, + { + "name": "Generate TAG DB Metadata", + "type": "SynapseNotebook", + "dependsOn": [ + { + "activity": "Convert TAG DB XML METADATA to JSON", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "notebook": { + "referenceName": "Purview_TAG_DB_Scan", + "type": "NotebookReference" + }, + "parameters": { + "out_file": { + "value": "pccsa_main/incoming", + "type": "string" + }, + "blob_container_name": { + "value": "pccsa", + "type": "string" + }, + "blob_account_name": { + "value": "", + "type": "string" + }, + "blob_relative_path": { + "value": "pccsa_main/tag-db-connector/tag-db-json", + "type": "string" + }, + "app_name": { + "value": "Purvew_TAG_DB_Scanner", + "type": "string" + }, + "blob_processed": { + "value": "pccsa_main/tag-db-connector/tag-db-processed", + "type": "string" + } + }, + "snapshot": true + } + } + ], + "parameters": { + "blob_xml_path": { + "type": "string", + "defaultValue": "pccsa_main/tag-db-connector/tag-db-xml" + } + }, + "folder": { + "name": "TAG-DB" + }, + "annotations": [] + } +} \ No newline at end of file diff --git a/examples/tag_db/tag_db.md b/examples/tag_db/tag_db.md new file mode 100644 index 0000000..c6aa606 --- /dev/null +++ b/examples/tag_db/tag_db.md @@ -0,0 +1,188 @@ + +# Tag DB Custom Connector + +This Sample will allow you to scan TAG DB metadata, exported from the TAG DB API. With some small changes you should be able to use the TAG DB SDK to direcly connect to the TAG DB and extract in real time. With this sample you will be able to: + +- Read TAG DB Metadata XML file and trransform into json +- Load into a folder to be processed by [Purview Custom Connector Solution Accelerator](https://github.com/microsoft/Purview-Custom-Connector-Solution-Accelerator), to be uploaded to Purview Catalog +- Monitor the process end to end +- Automate the metadata capture + - Scheduling + - Triggered by File event + +![Purview TAG DB Custom Metadata Scanner](../../assets/images/tag_db_diagram.svg) + +## Pull Tag DB Meta-data + +You should be able to export TAG DB metadata way to export Metadata from TAG DB Server (xml format) + +## Define the Tag DB Meta-model + +The TAG DB module was design following the hierarchical structure of the tags it is composed by: + +1. AFDatabase - it is the root and has one or more AFElement + 1. AFElement - Has relationship with multiples AFElement + 1. AFAttribute + 2. AFAnalysis + +![TAG DB Metadata Hierarchy](../../assets/images/AF_Hierarchy.svg) + +## Running the Example Solution + +### Prerequisites + +To use this custom connector you will need to deploy the complete 'Purview Custom Connector Solution Accelerator' and 'Purview-Custom-Types-Tool-Solution-Accelerator-main' that will allow you to create the Entity definition for TAG DB, also you will need a process that exports TAG DB metadata xml file. + +### Step 1. Configure Pipeline Parameters + +1. Deploy the 'Purview Custom Connector Solution Accelerator'. [Deploy Solution](../../purview_connector_services/deploy/deploy_sa.md) +2. Deploy the 'Purview Custom Types Tool Solution Accelerator' [Deploy Solution](https://github.com/microsoft/Purview-Custom-Types-Tool-Solution-Accelerator) +3. Deploy the 'TAG DB Connector'. [Deploy Solution](deploy/deploy_tag_db.md) + +### 2. Configure Development Environment + +1.Folder structure: + +![TAG DB Folder Structure](../../docs/images/tag_db_folder_structure.svg) + + 1.1 'tag-db-xml' - Location of the xml file exported from the TAG DB that will be used to load the tag db metadata into Azure Purview + 1.2 'tag-db-json' - Location where will be generated the conversion of the xml into a json format to be used to load meta data into Azure Purview + 1.3 'tag-db-purview-json' - Location where the 'TAG DB Scanner' generate the json file to be loaded int o operview + 1.4 'tag-db-proccessed' - Loacation where all files processed by 'TAG DB Scanner' will put all files processed by the solution. + +### Create types in Purview + +For general Atlas type information review the resource list in [README.MD](../../README.MD#purview-development-resources) + +* After configuring and starting the Purview Custom Types tool, you will be presented with a drop down allowing you to create a new service type (think of this like a grouping of all the types for a particular connector project), or a view of all types. We wil be creating new types for TAG DB, let select 'Create New Service Type' and on the next screen name it 'TAG DB' + ![pcttsa_select_service_type.png](../../assets/images/pcttsa_select_service_type.png) +* In the 'New Type Definition' Screen: + * Select 'Category' as Entity. + * Super Type = Dataset + * Name = 'afdatabase' + * Attibutes: + +| Attribute Name | Data Type | Cardinality | Optional? | Unique? | +| :---------------- | :-------- | :----------- | :----------- | :------ | +| DefaultPIServer | string | Single | not Optional | Unique | +| DefaultPIServerID | string | Single | not Optional | Unique | + +![AFDatabase](../../assets/images/crate_entity_afdatabase.svg) + +* Save to Azure + +* In the 'New Type Definition' Screen: + * Select 'Category' as Entity. + * Super Type = Dataset + * Name = 'afelement' + * Attibutes: + +| Attribute Name | Data Type | Cardinality | Optional? | Unique? | +| :---------------- | :-------- | :----------- | :----------- | :------ | +| Template | string | Single | not Optional | Unique | +| IsAnnotated | string | Single | not Optional | Unique | +| Modifier | string | Single | not Optional | Unique | +| Comment | string | Single | not Optional | Unique | +| EffectiveDate | string | Single | not Optional | Unique | +| ObsoleteDate | date | Single | not Optional | Unique | + +![AFElement](../../assets/images/afelement.svg) + +* Save to Azure + +* In the 'New Type Definition' Screen: + * Select 'Category' as Entity. + * Super Type = Dataset + * Name = 'afattribute' + * Attibutes: + +| Attribute Name | Data Type | Cardinality | Optional? | Unique? | +| :--------------------- | :-------- | :----------- | :----------- | :------ | +| IsHidden | int | Single | not Optional | Unique | +| IsManualDataEntry | int | Single | not Optional | Unique | +| IsConfigurationItem | int | Single | not Optional | Unique | +| IsExcluded | int | Single | not Optional | Unique | +| Trait | string | Single | is Optional | Unique | +| DefaultUOM | string | Single | is Optional | Unique | +| DisplayDigits | int | Single | not Optional | Unique | +| Type | string | Single | not Optional | Unique | +| TypeQualifier | string | Single | is Optional | Unique | +| DataReference | string | Single | is Optional | Unique | +| ConfigString | string | Single | is Optional | Unique | +| Value | string | Single | is Optional | Unique | +| AFAttributeCategoryRef | string | Single | is Optional | Unique | + +![AFAttribute](../../assets/images/afattribute.svg) + +* In the 'New Type Definition' Screen: + * Select 'Category' as Entity. + * Super Type = Dataset + * Name = 'afanalysis' + * Attibutes: + +| Attribute Name | Data Type | Cardinality | Optional? | Unique? | +| :--------------------- | :-------- | :----------- | :----------- | :------ | +| Template | string | Single | not Optional | Unique | +| CaseTemplate | string | Single | not Optional | Unique | +| OutputTime | string | Single | is Optional | Unique | +| Status | string | Single | not Optional | Unique | +| PublishResults | int | Single | not Optional | Unique | +| Priority | string | Single | not Optional | Unique | +| MaxQueueSize | int | Single | not Optional | Unique | +| GroupID | int | Single | not Optional | Unique | + +![AFAnalysis](../../assets/images/afanalysis.svg) + +* In the 'New Type Definition' Screen: + * Select 'Category' as Relationship: + * Relationship Category = Composition + * Name = 'afdatabase_afelement' + * Relationship: + + | Type | Name | Cardinality | Container? | Legacy? | + | :----------- | :-------- | :----------- | :----------- | :------ | + | afdatabase | Element | Set | true | false | + | afelement | Database | Single | false | false | + +![afdatabase_afelement](../../assets/images/afdatabase_afelement.svg) + +* In the 'New Type Definition' Screen: + * Select 'Category' as Relationship: + * Relationship Category = Composition + * Name = 'afelement_afelement' + * Relationship: + + | Type | Name | Cardinality | Container? | Legacy? | + | :----------- | :-------- | :----------- | :----------- | :------ | + | afelement | Child | Set | true | false | + | afelement | Parent | Single | false | false | + +![afdatabase_afelement](../../assets/images/afelement_afelement.svg) + +* In the 'New Type Definition' Screen: + * Select 'Category' as Relationship: + * Relationship Category = Composition + * Name = 'afelement_afattribute' + * Relationship: + + | Type | Name | Cardinality | Container? | Legacy? | + | :----------- | :------------- | :----------- | :----------- | :------ | + | afattribute | Parent Element | Single | false | false | + | afelement | Attribute | Set | true | false | + +![afdatabase_afelement](../../assets/images/afelement_afattribute.svg) + +* In the 'New Type Definition' Screen: + * Select 'Category' as Relationship: + * Relationship Category = Composition + * Name = 'afelement_afanalysis' + * Relationship: + + | Type | Name | Cardinality | Container? | Legacy? | + | :----------- | :---------------- | :----------- | :----------- | :------ | + | afanalysis | Reference Element | Single | false | false | + | afelement | Analysis | Set | true | false | + +![afdatabase_afelement](../../assets/images/afelement_afanalysis.svg) + +- The file xml sample metadata from TAG DB [tag-db-xml-sample.xml](./files/tag-db-xml-sample.xml) \ No newline at end of file diff --git a/purview_connector_services/Purview_Connector_Services.md b/purview_connector_services/Purview_Connector_Services.md new file mode 100644 index 0000000..abea35a --- /dev/null +++ b/purview_connector_services/Purview_Connector_Services.md @@ -0,0 +1,58 @@ +# Purview Connector Services + +![pccsa-design.svg](../assets/images/pccsa-design.svg) + +The Purview Connector Services are reusable services to convert templates partially generated by the Purview Custom Type Tool into entities in Purview + +## Infrastructure Design + +Services are implemented in the Purview_Load_Entity_Json notebook and are based on the PyApacheAtlas Python library + +The notebook is run from the Purview Load Custom Types pipeline which is triggered when entity definitions are created in blob storage (for example by the SSIS pipeline). The trigger looks for files created in the pccsa_main/incoming directory + +Processed entity definition files are delivered to the pccsa_main/outgoing directory + +## Features + +* The Purview Connector Services provide intelligent processing / ingestion of template types including: + * Translation of qualified names into Purview entities with associated unique identifiers (GUIDs) + * Creation of dummy types for lineage when sources appear in ETL activities which are not yet scanned + * Coordination of dummy types with newly scanned types - allowing for the deletion of dummy types when the source in the lineage relationship is scanned and actual entities become available + +_Purview SISS lineage example_ +![purview_ssis_lineage.svg](../assets/images/purview_ssis_lineage.png) + +_Purview Tag DB example_ +![purview_tag_db_properties.svg](../assets/images/purview_tag_db_properties.png) + +## Running the solution + +The connector services are the foundation of this accelerator. Connectors will re use these services to connect and ingest data into Purview. The connector does not run until the examples are installed, but there are some steps necessary to configure the connector for use. + +### Create Generic Type + +Using the Purview Custom Types Tool configured and installed using the steps in deploy_sa.md + +* Create a new Service Type Called 'Purview Custom Connector' then select that service type to enter the type creation experience. + +![pcttsa_select_service_type.svg](../assets/images/pcttsa_select_service_type.png) + +* Choose 'New Type Definition' and then fill in the resulting dialog as shown below. 'Save to Azure' when done + +![pcttsa_generic_entity.svg](../assets/images/pcttsa_generic_entity.png) + +### Configure Pipeline Trigger + +If you would like to trigger the pipeline based on incoming entity data, you need to enable it and then publish the change. This will trigger when files are added to the pccsa_main/incoming storage location + +* Go to the pipelines view in the Generic Connector pipeline. Click on Trigger and then New/Edit + +![synapse_edit_trigger.svg](../assets/images/synapse_edit_trigger.png) + +* Select the 'Trigger Load Custom Type' trigger to edit + +![synapse_select_trigger.svg](../assets/images/synapse_select_trigger.png) + +* At the bottom of the screen change the status to 'started' and then press 'continue' + +![synapse_start_trigger.svg](../assets/images/synapse_start_trigger.png) diff --git a/purview_connector_services/Synapse/linked_service/purviewaccws-WorkspaceDefaultStorage.json b/purview_connector_services/Synapse/linked_service/purviewaccws-WorkspaceDefaultStorage.json new file mode 100644 index 0000000..c1e7d7a --- /dev/null +++ b/purview_connector_services/Synapse/linked_service/purviewaccws-WorkspaceDefaultStorage.json @@ -0,0 +1,15 @@ +{ + "name": "purviewaccws-WorkspaceDefaultStorage", + "type": "Microsoft.Synapse/workspaces/linkedservices", + "properties": { + "typeProperties": { + "url": "https://.dfs.core.windows.net" + }, + "type": "AzureBlobFS", + "connectVia": { + "referenceName": "AutoResolveIntegrationRuntime", + "type": "IntegrationRuntimeReference" + }, + "annotations": [] + } +} \ No newline at end of file diff --git a/purview_connector_services/Synapse/notebook/Purview_Load_Entity.ipynb b/purview_connector_services/Synapse/notebook/Purview_Load_Entity.ipynb new file mode 100644 index 0000000..8844e88 --- /dev/null +++ b/purview_connector_services/Synapse/notebook/Purview_Load_Entity.ipynb @@ -0,0 +1,587 @@ +{ + "nbformat": 4, + "nbformat_minor": 2, + "metadata": { + "kernelspec": { + "name": "synapse_pyspark", + "display_name": "Synapse PySpark" + }, + "language_info": { + "name": "python" + }, + "save_output": true, + "synapse_widget": { + "version": "0.1", + "state": {} + } + }, + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "metadata": { + "tags": [ + "parameters" + ] + }, + "source": [ + "blob_container_name = \"\"\r\n", + "blob_account_name = \"\"\r\n", + "blob_relative_path = \"\"\r\n", + "app_name = \"\"\r\n", + "purview_name = \"\"\r\n", + "TENANT_ID = \"\"\r\n", + "CLIENT_ID = \"\"\r\n", + "CLIENT_SECRET = \"\"\r\n", + "blob_processed=\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + "import json\r\n", + "import os\r\n", + "import sys\r\n", + "from pyspark.sql.types import *\r\n", + "from pyspark.sql.functions import *\r\n", + "\r\n", + "\r\n", + "# PyApacheAtlas packages\r\n", + "# Connect to Atlas via a Service Principal\r\n", + "from pyapacheatlas.auth import ServicePrincipalAuthentication\r\n", + "from pyapacheatlas.core import PurviewClient, AtlasClassification, AtlasEntity, AtlasProcess, RelationshipTypeDef # Communicate with your Atlas server\r\n", + "from pyapacheatlas.readers import ExcelConfiguration, ExcelReader\r\n", + "from pyapacheatlas.core.util import GuidTracker,AtlasException\r\n", + "from pyapacheatlas.core import AtlasAttributeDef, AtlasEntity, PurviewClient\r\n", + "from pyapacheatlas.core.typedef import EntityTypeDef\r\n", + "from notebookutils import mssparkutils" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + "from pyspark.conf import SparkConf\r\n", + "from pyspark.sql import SparkSession\r\n", + "\r\n", + "my_jars = os.environ.get(\"SPARK_HOME\")\r\n", + "myconf = SparkConf()\r\n", + "myconf.setMaster(\"local\").setAppName(app_name)\r\n", + "myconf.set(\"spark.jars\",\"%s/jars/log4j-1.2.17.jar\" % my_jars)\r\n", + "spark = SparkSession\\\r\n", + " .builder\\\r\n", + " .appName(app_name)\\\r\n", + " .config(conf = myconf) \\\r\n", + " .getOrCreate()\r\n", + "\r\n", + "\r\n", + "Logger= spark._jvm.org.apache.log4j.Logger\r\n", + "mylogger = Logger.getLogger(app_name)\r\n", + "adls_home = 'abfss://%s@%s.dfs.core.windows.net/%s' % (blob_container_name, blob_account_name, blob_relative_path)\r\n", + "adls_processed = 'abfss://%s@%s.dfs.core.windows.net/%s' % (blob_container_name, blob_account_name, blob_processed)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + "def log_msgs(msg_type,msg):\r\n", + " \r\n", + " if msg_type.upper() == \"ERROR\":\r\n", + " print('ERROR: %s' % repr(msg))\r\n", + " mylogger.error(repr(msg))\r\n", + " else:\r\n", + " print('INFO: %s' % repr(msg))\r\n", + " mylogger.info(repr(msg))\r\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + " # Authenticate against your Atlas server\r\n", + "oauth = ServicePrincipalAuthentication(\r\n", + " tenant_id= TENANT_ID, \r\n", + " client_id=CLIENT_ID, \r\n", + " client_secret=CLIENT_SECRET \r\n", + " )\r\n", + "client = PurviewClient(\r\n", + " account_name = os.environ.get(\"PURVIEW_NAME\", purview_name),\r\n", + " authentication=oauth\r\n", + " )\r\n", + "gt = GuidTracker()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + "def Get_Rel_Inputs(json_obj):\r\n", + " try:\r\n", + " att_value = json_obj['relationshipAttributes']['inputs']\r\n", + " return True\r\n", + " except:\r\n", + " return False\r\n", + "\r\n", + "def Get_Rel_Outputs(json_obj):\r\n", + " try:\r\n", + " att_value = json_obj['relationshipAttributes']['outputs']\r\n", + " return True\r\n", + " except:\r\n", + " return False\r\n", + "\r\n", + "def Get_Outputs(json_obj):\r\n", + " try:\r\n", + " att_value = json_obj['attributes']['outputs']\r\n", + " return True\r\n", + " except:\r\n", + " return False\r\n", + "\r\n", + "def Get_Inputs(json_obj):\r\n", + " try:\r\n", + " att_value = json_obj['attributes']['inputs']\r\n", + " return True\r\n", + " except:\r\n", + " return False\r\n", + "def Get_Rel_Parents(json_obj):\r\n", + " try:\r\n", + " att_value = json_obj['relationshipAttributes']['parent']\r\n", + " return True\r\n", + " except:\r\n", + " return False" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + "def search_entities_byQuilifiedName(name):\r\n", + " results = client.search_entities('qualifiedName\\:%s' % name.replace(\":\",\"\\:\").replace(\"/\",\"\\/\").replace(\"{\",\"\\{\").replace(\"}\",\"\\}\"))\r\n", + " guid = None\r\n", + " for result in results:\r\n", + " if result['qualifiedName'] == name:\r\n", + " guid= result['id']\r\n", + " if result['entityType'] != 'purview_custom_connector_generic_entity':\r\n", + " #print(result)\r\n", + " break\r\n", + " return guid" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + "def removeDummyEntity(name):\r\n", + " try:\r\n", + " results = client.search_entities('qualifiedName\\:%s' % name.replace(\":\",\"\\:\").replace(\"/\",\"\\/\"))\r\n", + " for result in results:\r\n", + " if result['qualifiedName'] ==name:\r\n", + " entity = client.get_entity(guid=result['id'])\r\n", + " if len(entity['entities']) > 0 :\r\n", + " if len(entity['entities'][0]['relationshipAttributes']['inputToProcesses']) == 0 and len(entity['entities'][0]['relationshipAttributes']['outputFromProcesses']) == 0:\r\n", + " log_msgs(\"INFO\",'removeDummyEntity: Deleted Dummy entity%s' % (entity['entities'][0]))\r\n", + " client.delete_entity(entity['entities'][0]['guid'])\r\n", + " except:\r\n", + " log_msgs(\"ERROR\",'Build_Entity_Json: %s' % (str(e)))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + "def Create_Generic_Entity(dummy):\r\n", + " try:\r\n", + " log_msgs(\"INFO\",'Create_Generic_Entity - Dummy entity: %s' % dummy)\r\n", + " _qualifiedname = 'dummy://%s' % dummy['uniqueAttributes']['qualifiedName']\r\n", + " output_guid = search_entities_byQuilifiedName(_qualifiedname)\r\n", + " if output_guid==None:\r\n", + " attributes = {}\r\n", + " qualifiedName = dummy['uniqueAttributes']['qualifiedName']\r\n", + " attributes[\"purview_qualifiedName\"]= dummy['uniqueAttributes']['qualifiedName']\r\n", + " typeName = \"\"\r\n", + " if \"typeName\" in dummy['uniqueAttributes']:\r\n", + " typeName = dummy['uniqueAttributes']['typeName']\r\n", + " if \"source\" in dummy['uniqueAttributes']:\r\n", + " attributes[\"original_source\"]= dummy['uniqueAttributes']['source']\r\n", + " tmepname = qualifiedName.split('/')\r\n", + " Name= tmepname[len(tmepname)-1]\r\n", + " if \"Name\" in dummy['uniqueAttributes']:\r\n", + " Name = dummy['uniqueAttributes']['Name']\r\n", + " if \"name\" in dummy['uniqueAttributes']:\r\n", + " Name = dummy['uniqueAttributes']['name']\r\n", + "\r\n", + " generic_entity = AtlasEntity(\r\n", + " name = Name,\r\n", + " qualified_name = _qualifiedname,\r\n", + " typeName = \"purview_custom_connector_generic_entity\",\r\n", + " attributes = attributes,\r\n", + " guid = gt.get_guid()\r\n", + " )\r\n", + " upload_results = client.upload_entities(batch=[generic_entity])\r\n", + " if 'mutatedEntities' in upload_results:\r\n", + " if 'CREATE' in upload_results['mutatedEntities']:\r\n", + " if len(upload_results['mutatedEntities']['CREATE']) >0:\r\n", + " log_msgs(\"INFO\",'Create_Generic_Entity: Entities Created/Updated')\r\n", + " log_msgs(\"INFO\",'Create_Generic_Entity: %s' % upload_results)\r\n", + " return upload_results['mutatedEntities']['CREATE'][0]['guid']\r\n", + " else:\r\n", + " log_msgs(\"ERROR\",'Create_Generic_Entity: Fail to retrieve gui')\r\n", + " log_msgs(\"ERROR\",json.dump(upload_results))\r\n", + " return None\r\n", + " else:\r\n", + " log_msgs(\"ERROR\",'Create_Generic_Entity: Fail to retrieve CREATE')\r\n", + " log_msgs(\"ERROR\",json.dump(upload_results))\r\n", + " return None\r\n", + " log_msgs(\"ERROR\",'Create_Generic_Entity: Fail to retrieve mutatedEntities')\r\n", + " log_msgs(\"ERROR\", json.dump(upload_results))\r\n", + " return None\r\n", + " else:\r\n", + " return output_guid\r\n", + " except Exception as e:\r\n", + " log_msgs(\"ERROR\",'Create_Generic_Entity: %s' % (str(e)))\r\n", + " return None\r\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + "def Load_Entity_Json(json_file):\r\n", + " try:\r\n", + " purview_load_entities=[]\r\n", + " for i in json_file:\r\n", + " json_obj = json.loads(i)\r\n", + " json_obj\r\n", + " purview_load_entities.append(json_obj)\r\n", + " \r\n", + " upload_results = client.upload_entities(batch=purview_load_entities)\r\n", + " log_msgs(\"INFO\",'Entities Created/Updated')\r\n", + " #print(json.dumps(upload_results, indent=2))\r\n", + " return True\r\n", + " except Exception as e:\r\n", + " log_msgs(\"ERROR\",'Load_Entity_Json: %s' % (str(e)))\r\n", + " return False\r\n", + "\r\n", + "def Load_Entity_Json_fromJson(_json):\r\n", + " try:\r\n", + " purview_load_entities=[]\r\n", + " for i in _json:\r\n", + " purview_load_entities.append(i)\r\n", + " \r\n", + " log_msgs(\"INFO\",(purview_load_entities))\r\n", + " upload_results = client.upload_entities(batch=purview_load_entities)\r\n", + " log_msgs(\"INFO\",'Entities Created/Updated')\r\n", + "# print(json.dumps(upload_results, indent=2))\r\n", + " return True\r\n", + " except Exception as e:\r\n", + " log_msgs(\"ERROR\",'Load_Entity_Json_fromJson: %s' % (str(e)))\r\n", + " return False\r\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": false + }, + "source": [ + "def Build_Entity_Json(_json):\r\n", + " \r\n", + " _parent = gt.get_guid()\r\n", + " my_parents={}\r\n", + " final_json = []\r\n", + " rel_obj = {}\r\n", + " try:\r\n", + " for i in _json:\r\n", + " _qualifiedname=None\r\n", + " _typename = None\r\n", + " json_obj = json.loads(i)\r\n", + " try:\r\n", + " if 'typeName' in json_obj:\r\n", + " _typename = json_obj['typeName']\r\n", + " if _typename == None:\r\n", + " log_msgs('ERROR','JSON dont have typeName of the entity')\r\n", + " return False\r\n", + " except:\r\n", + " log_msgs('ERROR','JSON dont have typeName of the entity')\r\n", + " return False\r\n", + " try:\r\n", + " _qualifiedname = json_obj['attributes']['qualifiedName']\r\n", + " except:\r\n", + " log_msgs('ERROR','JSON dont have attributes/qualifiedName of the entity')\r\n", + " return False\r\n", + "\r\n", + " if _qualifiedname != None and _typename != None:\r\n", + " entity = client.get_entity(qualifiedName=_qualifiedname,typeName=_typename)\r\n", + " if len(entity) > 0:\r\n", + " entity_guid = entity[\"entities\"][0][\"guid\"]\r\n", + " print(entity_guid)\r\n", + " json_obj['guid'] = entity_guid\r\n", + " my_parents[_typename]=entity_guid\r\n", + " else:\r\n", + " if not 'guid' in json_obj:\r\n", + " json_obj['guid'] = gt.get_guid()\r\n", + " my_parents[_typename]=json_obj['guid']\r\n", + "\r\n", + " if Get_Outputs(json_obj):\r\n", + " for each in json_obj['attributes']['outputs']:\r\n", + " _qualifiedname = each['uniqueAttributes']['qualifiedName']\r\n", + " dummyEntities.append('dummy://%s' % _qualifiedname)\r\n", + " output_guid = search_entities_byQuilifiedName(_qualifiedname)\r\n", + " if output_guid==None:\r\n", + " refguid = Create_Generic_Entity(each) \r\n", + " if refguid != None:\r\n", + " each['uniqueAttributes']['guid'] = refguid\r\n", + " rel_obj[_qualifiedname] = refguid\r\n", + " else:\r\n", + " log_msgs(\"ERROR\",'Build_Entity_Json - output - results: Can\\'t Create Dummy '.join(each))\r\n", + " return False\r\n", + " else:\r\n", + " each['uniqueAttributes']['guid'] = output_guid\r\n", + " rel_obj[_qualifiedname] = output_guid\r\n", + "\r\n", + " if Get_Inputs(json_obj):\r\n", + " for each in json_obj['attributes']['inputs']:\r\n", + " _qualifiedname = each['uniqueAttributes']['qualifiedName']\r\n", + " dummyEntities.append('dummy://%s' % _qualifiedname)\r\n", + " input_guid = search_entities_byQuilifiedName(_qualifiedname)\r\n", + " if input_guid==None:\r\n", + " refguid = Create_Generic_Entity(each) \r\n", + " if refguid != None:\r\n", + " each['uniqueAttributes']['guid'] = refguid\r\n", + " rel_obj[_qualifiedname] = refguid\r\n", + " else:\r\n", + " log_msgs(\"ERROR\",'Build_Entity_Json - Input - results: Can\\'t Create Dummy '.join(each))\r\n", + " return False\r\n", + " else:\r\n", + " each['uniqueAttributes']['guid'] = input_guid\r\n", + " rel_obj[_qualifiedname] = input_guid\r\n", + "\r\n", + "\r\n", + " if Get_Rel_Inputs(json_obj):\r\n", + " for each in json_obj['relationshipAttributes']['inputs']:\r\n", + " _qualifiedname = each['qualifiedName']\r\n", + " each['guid'] = rel_obj[_qualifiedname]\r\n", + "\r\n", + " if Get_Rel_Outputs(json_obj):\r\n", + " for each in json_obj['relationshipAttributes']['outputs']:\r\n", + " _qualifiedname = each['qualifiedName']\r\n", + " each['guid'] = rel_obj[_qualifiedname]\r\n", + "\r\n", + " if Get_Rel_Parents(json_obj):\r\n", + " json_obj['relationshipAttributes']['parent']['guid'] = my_parents[json_obj['relationshipAttributes']['parent']['typeName']]\r\n", + " final_json.append(json_obj)\r\n", + " else:\r\n", + " return False\r\n", + " except Exception as e:\r\n", + " log_msgs(\"ERROR\",'Build_Entity_Json: %s' % (str(e)))\r\n", + " return False\r\n", + " try:\r\n", + " Load_Entity_Json_fromJson(final_json)\r\n", + " return True\r\n", + " except Exception as e:\r\n", + " log_msgs(\"ERROR\",'Build_Entity_Json: %s' % (str(e)))\r\n", + " return False" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "collapsed": true + }, + "source": [ + "havefiles = True\r\n", + "inicialnumfiles = 0\r\n", + "dummyEntities = []\r\n", + "while havefiles:\r\n", + " havefiles = False\r\n", + " files = mssparkutils.fs.ls(adls_home)\r\n", + " numoffiles = len(files)\r\n", + " processedfiles = 0\r\n", + " failfiles=0\r\n", + " for file in files:\r\n", + " if file.size > 0:\r\n", + " havefiles = True\r\n", + " i=0\r\n", + " filepath = \"\"\r\n", + " fileparts = file.path.split('/')\r\n", + " for filepart in fileparts:\r\n", + " if i < len(fileparts)-1:\r\n", + " filepath+='%s/' % filepart\r\n", + " i+=1\r\n", + " \r\n", + " filepath='%s/%s' % (adls_processed,file.name)\r\n", + " load_json = False\r\n", + " readComplexJSONDF=None\r\n", + " try:\r\n", + " readComplexJSONDF = spark.read.option(\"multiLine\",\"true\").json(file.path)\r\n", + " load_json=True\r\n", + " except Exception as e:\r\n", + " log_msgs('ERROR','Invalid Json: %s /r %s' % file.path,e.args[0])\r\n", + "\r\n", + " if load_json:\r\n", + " j = readComplexJSONDF.toJSON().collect()\r\n", + " log_msgs('INFO','Loading File: %s' % file.path)\r\n", + " if Build_Entity_Json(j):\r\n", + " try:\r\n", + " deletfile = mssparkutils.fs.rm(filepath)\r\n", + " \r\n", + " except:\r\n", + " log_msgs('INFO','No file to delete')\r\n", + " movefile = mssparkutils.fs.mv(src=file.path,dest=filepath)\r\n", + " processedfiles+=1\r\n", + " else:\r\n", + " failfiles+=1\r\n", + " if failfiles > 0 and processedfiles == 0:\r\n", + " print('Exit all files loaded')\r\n", + " break\r\n", + "if len(dummyEntities) > 0:\r\n", + " for dummyEntitie in dummyEntities:\r\n", + " removeDummyEntity(dummyEntitie)" + ] + } + ] +} \ No newline at end of file diff --git a/purview_connector_services/Synapse/pipeline/Purview Load Custom Types.json b/purview_connector_services/Synapse/pipeline/Purview Load Custom Types.json new file mode 100644 index 0000000..ecf4e03 --- /dev/null +++ b/purview_connector_services/Synapse/pipeline/Purview Load Custom Types.json @@ -0,0 +1,131 @@ +{ + "name": "Purview Load Custom Types", + "properties": { + "activities": [ + { + "name": "Purview Custom Types", + "type": "SynapseNotebook", + "dependsOn": [ + { + "activity": "AccSecret-to-var", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": true + }, + "userProperties": [], + "typeProperties": { + "notebook": { + "referenceName": "Purview_Load_Entity", + "type": "NotebookReference" + }, + "parameters": { + "blob_container_name": { + "value": "pccsa", + "type": "string" + }, + "blob_account_name": { + "value": "", + "type": "string" + }, + "blob_relative_path": { + "value": "pccsa_main/incoming", + "type": "string" + }, + "app_name": { + "value": "Purvew_Create_Entity_Def", + "type": "string" + }, + "purview_name": { + "value": "", + "type": "string" + }, + "TENANT_ID": { + "value": "", + "type": "string" + }, + "CLIENT_ID": { + "value": "", + "type": "string" + }, + "CLIENT_SECRET": { + "value": { + "value": "@variables('acc_secret')", + "type": "Expression" + }, + "type": "string" + }, + "blob_processed": { + "value": "pccsa_main/processed", + "type": "string" + } + }, + "snapshot": true + } + }, + { + "name": "kv-AccSecret", + "type": "WebActivity", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": true, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "url": "", + "connectVia": { + "referenceName": "AutoResolveIntegrationRuntime", + "type": "IntegrationRuntimeReference" + }, + "method": "GET", + "authentication": { + "type": "MSI", + "resource": "https://vault.azure.net" + } + } + }, + { + "name": "AccSecret-to-var", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "kv-AccSecret", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "variableName": "acc_secret", + "value": { + "value": "@activity('kv-AccSecret').output.value", + "type": "Expression" + } + } + } + ], + "variables": { + "acc_secret": { + "type": "String" + } + }, + "folder": { + "name": "Generic Connector" + }, + "annotations": [], + "lastPublishTime": "2021-05-14T21:45:04Z" + }, + "type": "Microsoft.Synapse/workspaces/pipelines" +} \ No newline at end of file diff --git a/purview_connector_services/Synapse/trigger/Trigger Load Custom Type.json b/purview_connector_services/Synapse/trigger/Trigger Load Custom Type.json new file mode 100644 index 0000000..200c1d2 --- /dev/null +++ b/purview_connector_services/Synapse/trigger/Trigger Load Custom Type.json @@ -0,0 +1,25 @@ +{ + "name": "Trigger Load Custom Type", + "properties": { + "annotations": [], + "runtimeState": "Started", + "pipelines": [ + { + "pipelineReference": { + "referenceName": "Purview Load Custom Types", + "type": "PipelineReference" + } + } + ], + "type": "BlobEventsTrigger", + "typeProperties": { + "blobPathBeginsWith": "/pccsa/blobs/pccsa_main/incoming", + "blobPathEndsWith": ".json", + "ignoreEmptyBlobs": true, + "scope": "/subscriptions//resourceGroups//providers/Microsoft.Storage/storageAccounts/", + "events": [ + "Microsoft.Storage.BlobCreated" + ] + } + } +} \ No newline at end of file diff --git a/purview_connector_services/deploy/.gitignore b/purview_connector_services/deploy/.gitignore new file mode 100644 index 0000000..4501d21 --- /dev/null +++ b/purview_connector_services/deploy/.gitignore @@ -0,0 +1,2 @@ +settings.sh +export_names.sh diff --git a/purview_connector_services/deploy/arm/deploy_purview.json b/purview_connector_services/deploy/arm/deploy_purview.json new file mode 100644 index 0000000..2d8f848 --- /dev/null +++ b/purview_connector_services/deploy/arm/deploy_purview.json @@ -0,0 +1,50 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + + "parameters": { + "purviewName": { + "type": "string" + } + }, + + "variables": { + "location": "[resourceGroup().location]" + }, + + "resources": [ + { + "name": "[parameters('purviewName')]", + "type": "Microsoft.Purview/accounts", + "apiVersion": "2020-12-01-preview", + "location": "[variables('location')]", + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "networkAcls": { + "defaultAction": "Allow" + } + }, + "dependsOn": [], + "sku": { + "name": "Standard", + "capacity": "4" + }, + "tags": {} + }, + { + "apiVersion": "2020-10-01", + "name": "pid-436d2bea-3759-4494-b63b-aa95d0407e1f", + "type": "Microsoft.Resources/deployments", + "properties": { + "mode": "Incremental", + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "resources": [] + } + } + } + ] +} \ No newline at end of file diff --git a/purview_connector_services/deploy/arm/deploy_storage.json b/purview_connector_services/deploy/arm/deploy_storage.json new file mode 100644 index 0000000..60f68fa --- /dev/null +++ b/purview_connector_services/deploy/arm/deploy_storage.json @@ -0,0 +1,114 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + + "parameters": { + "storageName": { + "type": "string" + }, + "synapseName": { + "type": "string" + } + }, + + "variables": { + "location": "[resourceGroup().location]", + "pccsaStorageName": "[replace(replace(toLower(parameters('storageName')),'-',''),'_','')]", + "pccsaStorageContainer": "pccsa", + "StorageBlobDataContributor": "ba92f5b4-2d11-453d-a403-e96b0029c9fe" + }, + + "resources": [ + { + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2019-06-01", + "name": "[variables('pccsaStorageName')]", + "location": "[variables('location')]", + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + }, + "kind": "StorageV2", + "properties": { + "isHnsEnabled": true, + "networkAcls": { + "bypass": "AzureServices", + "virtualNetworkRules": [], + "ipRules": [], + "defaultAction": "Allow" + }, + "supportsHttpsTrafficOnly": true, + "encryption": { + "services": { + "file": { + "enabled": true + }, + "blob": { + "enabled": true + } + }, + "keySource": "Microsoft.Storage" + }, + "accessTier": "Hot" + } + }, + { + "type": "Microsoft.Storage/storageAccounts/blobServices", + "apiVersion": "2019-06-01", + "name": "[concat(variables('pccsaStorageName'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', variables('pccsaStorageName'))]" + ], + "properties": { + "cors": { + "corsRules": [] + }, + "deleteRetentionPolicy": { + "enabled": false + } + } + }, + { + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2019-06-01", + "name": "[concat(variables('pccsaStorageName'), '/default/', variables('pccsaStorageContainer'))]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('pccsaStorageName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('pccsaStorageName'))]" + ], + "properties": { + "publicAccess": "None" + } + }, + { + "scope": "[concat('Microsoft.Storage/storageAccounts/', variables('pccsaStorageName'))]", + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2020-04-01-preview", + "name": "[guid(uniqueString(variables('pccsaStorageName')))]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', variables('pccsaStorageName'))]" + ], + "location": "[variables('location')]", + "properties": { + "roleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', variables('StorageBlobDataContributor'))]", + "principalId": "[reference(resourceId('Microsoft.Synapse/workspaces', parameters('synapseName')), '2019-06-01-preview', 'Full').identity.principalId]", + "principalType": "ServicePrincipal" + } + }, + { + "apiVersion": "2020-10-01", + "name": "pid-436d2bea-3759-4494-b63b-aa95d0407e1f", + "type": "Microsoft.Resources/deployments", + "properties": { + "mode": "Incremental", + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "resources": [] + } + } + } + ] +} + + diff --git a/purview_connector_services/deploy/arm/deploy_synapse.json b/purview_connector_services/deploy/arm/deploy_synapse.json new file mode 100644 index 0000000..1363d64 --- /dev/null +++ b/purview_connector_services/deploy/arm/deploy_synapse.json @@ -0,0 +1,197 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + + "parameters": { + "synapseName": { + "type": "string" + }, + "prefixName": { + "type": "string", + "maxLength": 7 + }, + "suffixName": { + "type": "string", + "maxLength": 9 + }, + "AllowAll": { + "type": "string", + "allowedValues": [ + "true", + "false" + ], + "defaultValue": "true" + } + }, + + "variables": { + "location": "[resourceGroup().location]", + "rgId": "[resourceGroup().id]", + "tenantId": "[subscription().tenantId]", + "paramName": "[parameters('prefixName')]", + "uniqueName": "[substring(uniqueString(variables('rgId')),0,4)]", + + "synapseWorkspaceName": "[parameters('synapseName')]", + "synapseStorageName": "[replace(replace(toLower(concat(concat('synapsestrg',variables('paramName')),variables('uniqueName'))),'-',''),'_','')]", + "synapseStorageContainer": "data", + "StorageBlobDataContributor": "ba92f5b4-2d11-453d-a403-e96b0029c9fe" + }, + + "resources": [ + { + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2019-06-01", + "name": "[variables('synapseStorageName')]", + "location": "[variables('location')]", + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + }, + "kind": "StorageV2", + "properties": { + "isHnsEnabled": true, + "networkAcls": { + "bypass": "AzureServices", + "virtualNetworkRules": [], + "ipRules": [], + "defaultAction": "Allow" + }, + "supportsHttpsTrafficOnly": true, + "encryption": { + "services": { + "file": { + "enabled": true + }, + "blob": { + "enabled": true + } + }, + "keySource": "Microsoft.Storage" + }, + "accessTier": "Hot" + } + }, + { + "type": "Microsoft.Storage/storageAccounts/blobServices", + "apiVersion": "2019-06-01", + "name": "[concat(variables('synapseStorageName'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', variables('synapseStorageName'))]" + ], + "properties": { + "cors": { + "corsRules": [] + }, + "deleteRetentionPolicy": { + "enabled": false + } + } + }, + { + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2019-06-01", + "name": "[concat(variables('synapseStorageName'), '/default/', variables('synapseStorageContainer'))]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('synapseStorageName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('synapseStorageName'))]" + ], + "properties": { + "publicAccess": "None" + } + }, + { + "type": "Microsoft.Synapse/workspaces", + "apiVersion": "2020-12-01", + "name": "[variables('synapseWorkspaceName')]", + "location": "[variables('location')]", + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "defaultDataLakeStorage": { + "accountUrl": "[concat('https://', variables('synapseStorageName') , '.dfs.core.windows.net')]", + "filesystem": "[variables('synapseStorageContainer')]" + }, + "virtualNetworkProfile": { + "computeSubnetId": "" + }, + "sqlAdministratorLogin": "sqladminuser" + }, + "resources": [ + { + "condition": "[equals(parameters('AllowAll'),'true')]", + "type": "firewallrules", + "apiVersion": "2019-06-01-preview", + "name": "allowAll", + "location": "[variables('location')]", + "dependsOn": [ "[variables('synapseWorkspaceName')]" ], + "properties": { + "startIpAddress": "0.0.0.0", + "endIpAddress": "255.255.255.255" + } + } + ] + }, + { + "type": "Microsoft.Synapse/workspaces/bigDataPools", + "apiVersion": "2020-12-01", + "name": "[concat(variables('synapseWorkspaceName'), '/spark1')]", + "location": "[variables('location')]", + "dependsOn": [ + "[resourceId('Microsoft.Synapse/workspaces', variables('synapseWorkspaceName'))]" + ], + "properties": { + "sparkVersion": "2.4", + "nodeCount": 3, + "nodeSize": "Medium", + "nodeSizeFamily": "MemoryOptimized", + "autoScale": { + "enabled": true, + "minNodeCount": 3, + "maxNodeCount": 6 + }, + "autoPause": { + "enabled": true, + "delayInMinutes": 15 + }, + "isComputeIsolationEnabled": false, + "sessionLevelPackagesEnabled": false, + "cacheSize": 0, + "dynamicExecutorAllocation": { + "enabled": true + }, + "provisioningState": "Succeeded" + } + }, + { + "scope": "[concat('Microsoft.Storage/storageAccounts/', variables('synapseStorageName'))]", + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2020-04-01-preview", + "name": "[guid(uniqueString(variables('synapseStorageName')))]", + "location": "[variables('location')]", + "dependsOn": [ + "[variables('synapseWorkspaceName')]" + ], + "properties": { + "roleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', variables('StorageBlobDataContributor'))]", + "principalId": "[reference(resourceId('Microsoft.Synapse/workspaces', variables('synapseWorkspaceName')), '2019-06-01-preview', 'Full').identity.principalId]", + "principalType": "ServicePrincipal" + } + }, + { + "apiVersion": "2020-10-01", + "name": "pid-436d2bea-3759-4494-b63b-aa95d0407e1f", + "type": "Microsoft.Resources/deployments", + "properties": { + "mode": "Incremental", + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "resources": [] + } + } + } + ] +} + + diff --git a/purview_connector_services/deploy/deploy_sa.md b/purview_connector_services/deploy/deploy_sa.md new file mode 100644 index 0000000..7fa891b --- /dev/null +++ b/purview_connector_services/deploy/deploy_sa.md @@ -0,0 +1,102 @@ +# Deployment of the Purview Custom Connector Solution Accelerator Base Services + +## Services Installed + + ![deployed services / changes](../../assets/images/service_deploy_block.svg) + +## Deployment Steps + +### Create an application identity and corresponding secret + + This will be the identity used for access to the Purview workspace from the Custom Type Tool application and from the synapse connector services. See [Create a service principal](https://docs.microsoft.com/en-us/azure/purview/tutorial-using-rest-apis#create-a-service-principal-application) + +### Clone the repository into Azure cloud shell + +* Start the cloud CLI in bash mode +* cd to the cloud storage directory (clouddrive) +* clone this repository into the clouddrive directory + +```bash +git clone https://github.com/microsoft/Purview-Custom-Connector-Solution-Accelerator.git + +``` + +### Configure application settings file + +* Download the settings.sh.rename file from the PurviewACC/purview_connector_services/deploy directory +* Modify the file as indicated to include a setup location, App name, client id, and secret +* Rename the file to settings.sh + +### Upload the application settings + +* Upload the settings.sh file (created above) to the deploy directory using the Upload/Download files tool + + ![upload files](../../assets/images/upload_with_cloud_console.png) + +* Choose the "Manage file share" option, navigate to the PurviewACC/purview_connector_services/deploy directory and copy the settings.sh file into this directory + + ![cloud console directory](../../assets/images/upload_file_to_cloud_console_directory.png) + + ![upload file dialog](../../assets/images/upload_file_dialog.png) + +### Run the deployment script + +* Navigate to the PurviewACC/purview_connector_services/deploy directory +* Run the deploy_sa.sh script + +```bash +./deploy_sa.sh + +``` + + For details about the scripts functionality, see [Reference - script actions](#reference---script-actions) + +### [Configure your Purview catalog to trust the service principal](https://docs.microsoft.com/en-us/azure/purview/tutorial-using-rest-apis#configure-your-catalog-to-trust-the-service-principal-application) + +* Open Purview Studio and select the Data Map icon in the left bar + + ![purview_root_collection.png](../../assets/images/purview_root_collection.png) + +* Choose the "View Details" link on the root collection + + ![purview_root_collection_detail.png](../../assets/images/purview_root_collection_detail.png) + +* Click on the 'Role assignments' tab in the root collection pane + + ![purview_root_collection_role_assignments.png](../../assets/images/purview_root_collection_role_assignments.png) + +* Click on the icon next to the role name and add the application identity you created above to the following roles: + * Data curators + * Data readers + +### Install the [Purview Custom Types Tool](https://github.com/microsoft/Purview-Custom-Types-Tool-Solution-Accelerator) + +* Follow the instructions in the project readme file +* You will need the app identity and secret you created above as well as information from the installed Purview service +* **_Note: If you are installing Node, be sure to install the LTS branch (v 14) NOT the latest (v 16)_** + +## Reference - script actions + +* Create resource group +* Deploy KeyVault + * Save client secret + * Save secret URL +* Deploy Purview + * Add app sp to purview roles +* Deploy Synapse + * Add Synapse to storage roles + * Add Synapse to retrieve KeyVault secrets + * Create linked service to storage + * Create spark pool + * Add package dependencies (PyApacheAtlas) + * Import notebooks + * Import pipelines + * Import trigger +* Deploy Storage Account + * Create folder structure + * Save storage account key to KeyVault secret +* Write output name variables to file for use in other deployments + +## Privacy + +To opt out of information collection as described in [privacy.md](../../PRIVACY.md), remove the GUID section from all templates in the PurviewACC/purview_connector_services/deploy/arm directory diff --git a/purview_connector_services/deploy/deploy_sa.sh b/purview_connector_services/deploy/deploy_sa.sh new file mode 100644 index 0000000..f6a014b --- /dev/null +++ b/purview_connector_services/deploy/deploy_sa.sh @@ -0,0 +1,162 @@ +#!/bin/bash + +# This script can take over 1 hour to complete +# This script requires contributor and user access administrator permissions to run +source ./settings.sh + +# To run in Azure Cloud CLI, comment this section out +# az login --output none +# az account set --subscription "Early Access Engineering Subscription" --output none + +# dynamically load missing az dependancies without prompting +az config set extension.use_dynamic_install=yes_without_prompt --output none + +# Parameters +base="pccsa" # must be < 7 chars, all letters + +# Generate a unique suffix for the service names +let svc_suffix=$RANDOM*$RANDOM + +# Names +synapse_name=$base"synapse"$svc_suffix +purview_name=$base"purview"$svc_suffix +storage_name=$base"storage"$svc_suffix +resource_group=$base"_rg_"$svc_suffix +key_vault_name=$base"keyvault"$svc_suffix + +# Retrieve account info +tenant_id=$(az account show --query "homeTenantId" -o tsv) +subscription_id=$(az account show --query "id" -o tsv) + +################################################################################# + +echo "Creating resource group $resource_group" +az group create --name $resource_group --location $location --output none >> log_connector_services_deploy.txt + + +################################################################################# + +echo "Adding keyvault and client secret" +# Create KeyVault and generate secrets +az keyvault create \ + --name $key_vault_name \ + --resource-group $resource_group \ + --location $location \ + --enabled-for-template-deployment true\ + --output none +az keyvault secret set --vault-name $key_vault_name --name client-secret --value $client_secret --output none >> log_connector_services_deploy.txt + +echo "Retrieving client secret uri" +# Get secret URI to fill in pipeline templates later +client_secret_uri=$(az keyvault secret show --name client-secret --vault-name $key_vault_name --query 'id' -o tsv) + +################################################################################## + +echo "Starting Purview template deployment" +# Use template for deployment +params="{\"purviewName\":{\"value\":\"$purview_name\"}}" +az deployment group create --resource-group $resource_group --parameters $params --template-file ./arm/deploy_purview.json --output none >> log_connector_services_deploy.txt + +echo "Adding app sp $app_object_id to Purview curator and reader roles" +app_object_id=$(az ad sp list --display-name $client_name --query "[0].objectId" -o tsv) +echo "App object id: $app_object_id" +purview_resource="/subscriptions/$subscription_id/resourcegroups/$resource_group/providers/Microsoft.Purview/accounts/$purview_name" +purview_data_curator_id=$(az role definition list --name "Purview Data Curator" --query "[].{name:name}" -o tsv) +purview_data_reader_id=$(az role definition list --name "Purview Data Reader" --query "[].{name:name}" -o tsv) +echo "purview_data_curator_id is $purview_data_curator_id" >> log_connector_services_deploy.txt +echo "purview_data_reader_id is $purview_data_reader_id" >> log_connector_services_deploy.txt +# Add client app id to purview data curator and reader roles +# https://docs.microsoft.com/en-us/azure/role-based-access-control/role-assignments-cli +az role assignment create --assignee $app_object_id --role $purview_data_curator_id --scope $purview_resource --output none >> log_connector_services_deploy.txt +az role assignment create --assignee $app_object_id --role $purview_data_reader_id --scope $purview_resource --output none >> log_connector_services_deploy.txt + +############################################################################### +echo "Deploying Synapse ARM Template" +# Use template for deployment +params="{\"prefixName\":{\"value\":\"$base\"},"\ +"\"suffixName\":{\"value\":\"$svc_suffix\"},"\ +"\"synapseName\":{\"value\":\"$synapse_name\"}}" +az deployment group create --resource-group $resource_group --parameters $params --template-file ./arm/deploy_synapse.json --output none >> log_connector_services_deploy.txt + +echo "Setting KeyVault secret policy for Synapse" +# Allow synapse pipelines (MIP) to retrieve keyvault secrets +synapse_sp_id=$(az synapse workspace show --resource-group $resource_group --name $synapse_name --query 'identity.principalId' -o tsv) +az keyvault set-policy -n $key_vault_name --secret-permissions get list --object-id $synapse_sp_id --output none >> log_connector_services_deploy.txt + +echo "Creating linked services in Synapse" +# Configure Synapse Notebooks and Pipelines +# Add storage account name to linked service json +sed "s//$storage_name/g" ../synapse/linked_service/purviewaccws-WorkspaceDefaultStorage.json > ../synapse/linked_service/purviewaccws-WorkspaceDefaultStorage-tmp.json +# Create linked service in Synapse +az synapse linked-service create --workspace-name $synapse_name --name purviewaccws-WorkspaceDefaultStorage --file @../synapse/linked_service/purviewaccws-WorkspaceDefaultStorage-tmp.json --output none >> log_connector_services_deploy.txt +# Delete the tmp json with the added storage account name +rm ../synapse/linked_service/purviewaccws-WorkspaceDefaultStorage-tmp.json + +echo "Creating spark pool" +# Create SPARK pool +az synapse spark pool create --name notebookrun --workspace-name $synapse_name --resource-group $resource_group --spark-version 2.4 --node-count 10 --node-size Medium --delay 10 --enable-auto-pause true --output none >> log_connector_services_deploy.txt +# Add packages +az synapse spark pool update --name notebookrun --workspace-name $synapse_name --resource-group $resource_group --library-requirements ./requirements.txt --output none >> log_connector_services_deploy.txt + +echo "Creating Synapse notebooks" +# Note: add notebooks and attach to sparkpool +az synapse notebook create --workspace-name $synapse_name --spark-pool-name notebookrun --name Purview_Load_Entity --file @../synapse/notebook/Purview_Load_Entity.ipynb --output none + +echo "Creating Synapse pipelines" +# Build multiple substitutions for pipeline json - note sed can use any delimeter so url changed to avoid conflict with slash char +pipeline_sub="s//$storage_name/;"\ +"s//$purview_name/;"\ +"s//$tenant_id/;"\ +"s//$client_id/;"\ +"s@@$client_secret_uri?api-version=7.0@" +sed $pipeline_sub '../synapse/pipeline/Purview Load Custom Types.json' > '../synapse/pipeline/Purview Load Custom Types-tmp.json' +# Create pipeline in Synapse +az synapse pipeline create --workspace-name $synapse_name --name 'Purview Load Custom Types' --file '@../synapse/pipeline/Purview Load Custom Types-tmp.json' --output none >> log_connector_services_deploy.txt +# Delete the tmp json +rm '../synapse/pipeline/Purview Load Custom Types-tmp.json' + +echo "Creating Synapse pipeline trigger" +# Build multiple substitutions for trigger json +pipeline_sub="s//$subscription_id/;"\ +"s//$resource_group/;"\ +"s//$storage_name/" +sed $pipeline_sub '../synapse/trigger/Trigger Load Custom Type.json' > '../synapse/trigger/Trigger Load Custom Type-tmp.json' +# Create trigger in Synapse +az synapse trigger create --workspace-name $synapse_name --name 'Trigger Load Custom Type' --file '@../synapse/trigger/Trigger Load Custom Type-tmp.json' --output none >> log_connector_services_deploy.txt +# Delete the tmp json +rm '../synapse/trigger/Trigger Load Custom Type-tmp.json' + +################################################################################## + +echo "Deploying Storage ARM Template" +# Use template for deployment +params="{\"synapseName\":{\"value\":\"$synapse_name\"},"\ +"\"storageName\":{\"value\":\"$storage_name\"}}" +az deployment group create --resource-group $resource_group --parameters $params --template-file ./arm/deploy_storage.json --output none >> log_connector_services_deploy.txt + +echo "Creating folder structure in ADLS" +# Create storage dir structure +mkdir pccsa_main; cd pccsa_main; mkdir incoming; touch ./incoming/tmp; mkdir processed; touch ./processed/tmp; cd .. +# Upload dir structure to storage +az storage fs directory upload -f pccsa --account-name $storage_name -s ./pccsa_main -d . --recursive --output none >> log_connector_services_deploy.txt +# Remove tmp directory structure +rm -r pccsa_main +# Get storage account key and add to keyvault secrets +storage_account_key=$(az storage account keys list --resource-group $resource_group --account-name $storage_name --query [0].value -o tsv) +az keyvault secret set --vault-name $key_vault_name --name storage-account-key --value $storage_account_key --output none >> log_connector_services_deploy.txt + +################################################################################## +echo "Writing name output" +# Write names into bash script so they can be sourced into example deploy scripts +echo '#!/bin/bash' > ./export_names.sh +echo >> ./export_names.sh +echo "key_vault_name=$key_vault_name" >> ./export_names.sh +echo "synapse_name=$synapse_name" >> ./export_names.sh +echo "purview_name=$purview_name" >> ./export_names.sh +echo "storage_name=$storage_name" >> ./export_names.sh +echo "tenant_id=$tenant_id" >> ./export_names.sh +echo "subscription_id=$subscription_id" >> ./export_names.sh +echo "resource_group=$resource_group" >> ./export_names.sh +echo "prefix=$base" >> ./export_names.sh +echo "suffix=$svc_suffix" >> ./export_names.sh + diff --git a/purview_connector_services/deploy/requirements.txt b/purview_connector_services/deploy/requirements.txt new file mode 100644 index 0000000..d93e005 --- /dev/null +++ b/purview_connector_services/deploy/requirements.txt @@ -0,0 +1 @@ +pyapacheatlas \ No newline at end of file diff --git a/purview_connector_services/deploy/settings.sh.rename b/purview_connector_services/deploy/settings.sh.rename new file mode 100644 index 0000000..873c8f8 --- /dev/null +++ b/purview_connector_services/deploy/settings.sh.rename @@ -0,0 +1,8 @@ +#!/bin/bash + +# Fill values in here. They will not be synced with repo +# and will be sourced into the main bash script. +location="" +client_name="" +client_id="" +client_secret="" \ No newline at end of file