1
2
3
4
5 package org.directdemocracyportal.democracy.service.governmentloader;
6
7 import java.net.MalformedURLException;
8 import java.net.URL;
9 import java.util.ArrayList;
10 import java.util.HashSet;
11 import java.util.List;
12 import java.util.Set;
13
14 import org.apache.commons.logging.Log;
15 import org.apache.commons.logging.LogFactory;
16 import org.directdemocracyportal.democracy.model.core.Role;
17 import org.directdemocracyportal.democracy.model.world.Country;
18 import org.directdemocracyportal.democracy.model.world.Election;
19 import org.directdemocracyportal.democracy.model.world.ElectionType;
20 import org.directdemocracyportal.democracy.model.world.Government;
21 import org.directdemocracyportal.democracy.model.world.GovernmentType;
22 import org.directdemocracyportal.democracy.model.world.Organisation;
23 import org.directdemocracyportal.democracy.model.world.OrganisationType;
24 import org.directdemocracyportal.democracy.model.world.Person;
25 import org.directdemocracyportal.democracy.model.world.PoliticalParty;
26 import org.directdemocracyportal.democracy.model.world.Region;
27 import org.directdemocracyportal.democracy.model.world.RegionType;
28 import org.directdemocracyportal.democracy.service.OrganisationAlreadyExistException;
29 import org.directdemocracyportal.democracy.service.PortalService;
30
31 import com.gargoylesoftware.htmlunit.WebClient;
32 import com.gargoylesoftware.htmlunit.html.HtmlAnchor;
33 import com.gargoylesoftware.htmlunit.html.HtmlElement;
34 import com.gargoylesoftware.htmlunit.html.HtmlPage;
35 import com.gargoylesoftware.htmlunit.html.HtmlTable;
36 import com.gargoylesoftware.htmlunit.html.HtmlTableBody;
37 import com.gargoylesoftware.htmlunit.html.HtmlTableCell;
38 import com.gargoylesoftware.htmlunit.html.HtmlTableRow;
39
40 /***
41 * The Class SwedishGovernmentImporterImpl.
42 */
43 public class SwedishGovernmentImporterImpl implements GovernmentImporter
44 {
45
46 /*** The Constant SVERIGE. */
47 private static final String SVERIGE = "Sverige";
48
49 /*** The Constant SVERIGES_RIKSDAG. */
50 public static final String SVERIGES_RIKSDAG = "Sveriges Riksdag";
51
52 /*** The log. */
53 private static Log log = LogFactory
54 .getLog(SwedishGovernmentImporterImpl.class);
55
56 /*** The web client. */
57 private final WebClient webClient;
58
59 /*** The url. */
60 private final URL url;
61
62 /*** The portal service. */
63 private final PortalService portalService;
64
65 /***
66 * Instantiates a new swedish government importer impl.
67 *
68 * @param webClient the web client
69 * @param url the url
70 * @param portalService the portal service
71 * @throws MalformedURLException the malformed url exception
72 */
73 public SwedishGovernmentImporterImpl(WebClient webClient, String url,PortalService portalService) throws MalformedURLException {
74 this.webClient = webClient;
75 this.portalService = portalService;
76 this.url = new URL(url);
77 }
78
79
80
81
82
83
84 @SuppressWarnings("unchecked")
85 public void doImport() {
86
87 log.info("Starting import");
88
89 Country sweden = portalService.findCountryByName(SVERIGE);
90 if (sweden == null) {
91
92 sweden = portalService.createCountry(SVERIGE);
93
94 Government government = portalService.createGovernment(
95 "Konungariket Sverige", sweden.getId(),
96 GovernmentType.Constitutional_Monarchy,"Carl Gustaf Folke Hubertus");
97
98 portalService.createElection("Sveriges Riksdags val",ElectionType.National,government.getId());
99
100 for (ActivePoliticalParty party : getCurrentActivePartyList()) {
101 PoliticalParty politicalParty = portalService
102 .createPoliticalPartyById(party.name, party.shortCode,
103 sweden.getId());
104 }
105
106 try {
107 portalService.createOrganisationById("Regeringskansliet", null,
108 sweden.getId(),sweden.getId(), OrganisationType.National, government.getId());
109 } catch (OrganisationAlreadyExistException e2) {
110
111 e2.printStackTrace();
112 }
113
114 Organisation parliament=null;
115 try {
116 parliament = portalService.createOrganisation(
117 SVERIGES_RIKSDAG, null, sweden,
118 sweden, OrganisationType.National, government);
119 } catch (OrganisationAlreadyExistException e2) {
120
121 e2.printStackTrace();
122 }
123 Role ledamot = portalService.createRoleInOrg("Ledamot", parliament);
124
125 List<Politician> currentList;
126 try {
127 currentList = getCurrentList();
128 } catch (Exception e) {
129 log.error("Problem importing swedish parliament members", e);
130 return;
131 }
132
133 log.info("Importing members in " + parliament.getName());
134
135 for (Politician member : currentList) {
136 PoliticalParty politicalParty = portalService.findPartyByShortCode(member.getParty().toLowerCase().trim());
137
138 Person person= portalService.createPerson(member.getName());
139 portalService.addMemberById(politicalParty.getId(), person.getId());
140 portalService.addMemberById(parliament.getId(), person.getId());
141 portalService.addRolePlayedById(ledamot.getId(), person.getId());
142 }
143
144
145 try {
146 importRegiononalGovernments(government, sweden);
147 } catch (Exception e1) {
148
149 e1.printStackTrace();
150 }
151
152
153 try {
154 importRegionsAndMunicipals(government, sweden);
155 } catch (Exception e) {
156
157 e.printStackTrace();
158 }
159
160 try {
161 importInternationalGovernments(government, sweden);
162 } catch (Exception e) {
163
164 e.printStackTrace();
165 }
166 }
167 log.info("Import completed");
168 }
169
170 /***
171 * Gets the current list.
172 *
173 * @return the current list
174 * @throws Exception the exception
175 */
176 private List<Politician> getCurrentList() throws Exception {
177 List<Politician> members = new ArrayList<Politician>();
178
179 HtmlPage page = (HtmlPage) webClient.getPage(url);
180 HtmlElement pageContent = page.getDocumentElement();
181 HtmlElement parliamentTable = pageContent
182 .getHtmlElementsByTagName("table").get(0);
183 List<HtmlElement> htmlElementsByTagName2 = parliamentTable
184 .getHtmlElementsByTagName("span");
185 for (HtmlElement span : htmlElementsByTagName2) {
186 HtmlAnchor homePage = (HtmlAnchor) span.getFirstChild();
187 String name = homePage.asText();
188 String href = homePage.getHrefAttribute();
189 String text = span.asText();
190 String party = text.substring(text.indexOf('(') + 1, text
191 .indexOf(')'));
192 String region = text.substring(text.indexOf(')') + 1,
193 text.lastIndexOf(',')).trim();
194 String role = text.substring(text.lastIndexOf(',') + 1,
195 text.length()).trim();
196 members.add(new Politician(name, href, party, region, role));
197 }
198 return members;
199 }
200
201 /***
202 * The Class Politician.
203 */
204 private class Politician
205 {
206
207 /*** The name. */
208 private final String name;
209
210 /*** The href. */
211 private final String href;
212
213 /*** The party. */
214 private final String party;
215
216 /*** The region. */
217 private final String region;
218
219 /*** The role. */
220 public final String role;
221
222 /***
223 * Instantiates a new politician.
224 *
225 * @param name the name
226 * @param href the href
227 * @param party the party
228 * @param region the region
229 * @param role the role
230 */
231 public Politician(String name, String href, String party,
232 String region, String role) {
233 super();
234 this.name = name.trim();
235 this.href = href.trim();
236 this.party = party.trim().toLowerCase();
237 this.region = region.trim();
238 this.role = role.trim();
239 }
240
241 /***
242 * Gets the name.
243 *
244 * @return the name
245 */
246 public String getName() {
247 return name;
248 }
249
250 /***
251 * Gets the href.
252 *
253 * @return the href
254 */
255 public String getHref() {
256 return href;
257 }
258
259 /***
260 * Gets the party.
261 *
262 * @return the party
263 */
264 public String getParty() {
265 return party;
266 }
267
268 /***
269 * Gets the region.
270 *
271 * @return the region
272 */
273 public String getRegion() {
274 return region;
275 }
276
277 /***
278 * Gets the role.
279 *
280 * @return the role
281 */
282 public String getRole() {
283 return role;
284 }
285 }
286
287 /***
288 * The Class ActivePoliticalParty.
289 */
290 private class ActivePoliticalParty
291 {
292
293 /*** The name. */
294 private final String name;
295
296 /*** The short code. */
297 private final String shortCode;
298
299 /***
300 * Instantiates a new active political party.
301 *
302 * @param name the name
303 * @param shortCode the short code
304 */
305 public ActivePoliticalParty(String name, String shortCode) {
306 this.name = name.trim();
307 this.shortCode = shortCode.trim().toLowerCase();
308 }
309
310 /***
311 * Gets the name.
312 *
313 * @return the name
314 */
315 public String getName() {
316 return name;
317 }
318
319
320
321
322
323
324 @Override
325 public int hashCode() {
326 final int prime = 31;
327 int result = 1;
328 result = prime * result + ((name == null) ? 0 : name.hashCode());
329 result = prime * result
330 + ((shortCode == null) ? 0 : shortCode.hashCode());
331 return result;
332 }
333
334
335
336
337
338
339 @Override
340 public boolean equals(Object obj) {
341 if (this == obj)
342 return true;
343 if (obj == null)
344 return false;
345 if (getClass() != obj.getClass())
346 return false;
347 final ActivePoliticalParty other = (ActivePoliticalParty) obj;
348 if (name == null) {
349 if (other.name != null)
350 return false;
351 } else if (!name.equals(other.name))
352 return false;
353 if (shortCode == null) {
354 if (other.shortCode != null)
355 return false;
356 } else if (!shortCode.equals(other.shortCode))
357 return false;
358 return true;
359 }
360 }
361
362 /***
363 * Gets the current active party list.
364 *
365 * @return the current active party list
366 */
367 private Set<ActivePoliticalParty> getCurrentActivePartyList() {
368 Set<ActivePoliticalParty> parties = new HashSet<ActivePoliticalParty>();
369
370 try {
371 HtmlPage nationalParties = (HtmlPage) webClient
372 .getPage("http://www.val.se/val/val2006/slutlig/R/rike/roster.html");
373 parties.addAll(getPoliticalPartiesFromResultPage(nationalParties));
374
375 HtmlPage regionalParties = (HtmlPage) webClient
376 .getPage("http://www.val.se/val/val2006/slutlig/L/rike/delar.html");
377 parties.addAll(getPoliticalPartiesForRegionalList(regionalParties));
378
379 HtmlPage localParties = (HtmlPage) webClient
380 .getPage("http://www.val.se/val/val2006/slutlig/K/rike/delar.html");
381 parties.addAll(getPoliticalPartiesForRegionalList(localParties));
382
383
384 parties.add(new ActivePoliticalParty("Junilistan", "jl"));
385
386 } catch (Exception e) {
387 e.printStackTrace();
388 }
389 return parties;
390 }
391
392 /***
393 * Gets the political parties from result page.
394 *
395 * @param page the page
396 * @return the political parties from result page
397 * @throws Exception the exception
398 */
399 @SuppressWarnings("unchecked")
400 private Set<ActivePoliticalParty> getPoliticalPartiesFromResultPage(
401 HtmlPage page) throws Exception {
402 Set<ActivePoliticalParty> parties = new HashSet<ActivePoliticalParty>();
403
404 HtmlElement pageContent = page.getDocumentElement();
405 HtmlTable table = (HtmlTable) pageContent.getElementsByAttribute(
406 "table", "class", "rostfordelning").get(0);
407 List<HtmlTableRow> rows = new ArrayList(table.getRows());
408
409 rows.remove(0);
410 rows.remove(0);
411 rows.remove(rows.size() - 1);
412
413 for (HtmlTableRow row : rows) {
414 int mandat = Integer.valueOf(((HtmlElement) row.getCells().get(4))
415 .asText().trim());
416
417 if (mandat != 0) {
418 ActivePoliticalParty activePoliticalParty = new ActivePoliticalParty(
419 ((HtmlElement) row.getCells().get(1)).asText().trim(),
420 ((HtmlElement) row.getCells().get(0)).asText()
421 .toLowerCase().trim());
422 parties.add(activePoliticalParty);
423 }
424 }
425 return parties;
426 }
427
428 /***
429 * Gets the political parties for regional list.
430 *
431 * @param page the page
432 * @return the political parties for regional list
433 * @throws Exception the exception
434 */
435 @SuppressWarnings("unchecked")
436 private Set<ActivePoliticalParty> getPoliticalPartiesForRegionalList(
437 HtmlPage page) throws Exception {
438 Set<ActivePoliticalParty> parties = new HashSet<ActivePoliticalParty>();
439 HtmlElement pageContent = page.getDocumentElement();
440 HtmlElement element = pageContent.getElementById("delar");
441 List<HtmlAnchor> htmlElementsByTagName = element
442 .getHtmlElementsByTagName("a");
443
444 for (HtmlAnchor a : htmlElementsByTagName) {
445 parties.addAll(getPoliticalPartiesFromResultPage((HtmlPage) a
446 .openLinkInNewWindow()));
447 }
448 return parties;
449 }
450
451 /***
452 * Import regions and municipals.
453 *
454 * @param government the government
455 * @param country the country
456 * @throws Exception the exception
457 */
458 @SuppressWarnings("unchecked")
459 public void importRegionsAndMunicipals(Government government,
460 Country country) throws Exception {
461 HtmlPage page = (HtmlPage) webClient
462 .getPage("http://www.val.se/val/val2006/valdaled/index_kommun.html");
463 HtmlElement pageContent = page.getDocumentElement();
464 HtmlElement div = pageContent.getElementById("sik");
465
466 List<HtmlElement> htmlElementsByTagName = div
467 .getHtmlElementsByTagName("a");
468
469 for (HtmlElement element : htmlElementsByTagName) {
470 HtmlAnchor anchor = (HtmlAnchor) element;
471 ImportAllMunicipalsInRegion(anchor, government, country);
472 }
473 }
474
475 /***
476 * Import regiononal governments.
477 *
478 * @param government the government
479 * @param country the country
480 * @throws Exception the exception
481 */
482 @SuppressWarnings("unchecked")
483 public void importRegiononalGovernments(Government government,
484 Country country) throws Exception {
485 HtmlPage page = (HtmlPage) webClient
486 .getPage("http://www.val.se/val/val2006/valdaled/index_landsting.html");
487 HtmlElement pageContent = page.getDocumentElement();
488 HtmlElement div = pageContent.getElementById("sik");
489
490 List<HtmlElement> htmlElementsByTagName = div
491 .getHtmlElementsByTagName("a");
492
493 for (HtmlElement element : htmlElementsByTagName) {
494 HtmlAnchor anchor = (HtmlAnchor) element;
495
496 importRegionalGovernment(anchor, government, country);
497 }
498 }
499
500 /***
501 * Import regional government.
502 *
503 * @param anchor the anchor
504 * @param government the government
505 * @param country the country
506 * @throws Exception the exception
507 */
508 private void importRegionalGovernment(HtmlAnchor anchor,
509 Government government, Country country) throws Exception {
510 HtmlPage page = (HtmlPage) anchor.openLinkInNewWindow();
511 HtmlElement pageContent = page.getDocumentElement();
512
513 String regionName = anchor.asText().trim().replace("s landsting", "")
514 .trim();
515
516 Region region = new Region();
517 region.setParent(country);
518 country.getParts().add(region);
519 region.setRegionType(RegionType.Regional);
520 region.setName(regionName);
521
522
523 Government regionalGovernment = portalService.createRegionalGovernment(
524 anchor.asText(), country, region, OrganisationType.Regional,
525 government);
526
527 Election election = new Election();
528 election.setName(regionalGovernment.getName() + "sval");
529 election.setType(ElectionType.Regional);
530 election.setGovernment(regionalGovernment);
531 regionalGovernment.getElections().add(election);
532
533
534 portalService.createOrganisation("Landstingsstyrelse "
535 + region.getName() + " landsting", null, country,
536 region, OrganisationType.Regional, regionalGovernment);
537
538 Organisation regionalParliament = portalService.createOrganisation(
539 "Landstingsfullmäktige " + region.getName() + " landsting",
540 null, country, region, OrganisationType.Regional, regionalGovernment);
541
542 Role landstingFullmäktige = portalService.createRoleInOrg(
543 "Landstingsfullmäktige", regionalParliament);
544
545 log.info("Importing members in " + regionalParliament.getName());
546
547 List<Politician> members = getAllRegionalGovernmentMembers(pageContent);
548 for (Politician member : members) {
549 PoliticalParty politicalParty = portalService.findPartyByShortCode(member.getParty().toLowerCase().trim());
550 Person person = portalService.createPerson(member.getName());
551
552 portalService.addMemberById(politicalParty.getId(), person.getId());
553 portalService.addMemberById(regionalParliament.getId(), person.getId());
554 portalService.addRolePlayedById(landstingFullmäktige.getId(), person.getId());
555 }
556 }
557
558 /***
559 * Import all municipals in region.
560 *
561 * @param anchor the anchor
562 * @param government the government
563 * @param country the country
564 * @throws Exception the exception
565 */
566 @SuppressWarnings("unchecked")
567 private void ImportAllMunicipalsInRegion(HtmlAnchor anchor,
568 Government government, Country country) throws Exception {
569 if (anchor.asText().equals("Gotland")) {
570 ImportMunicipal(anchor, government, country, country);
571 } else {
572
573 HtmlPage page = (HtmlPage) anchor.openLinkInNewWindow();
574 HtmlElement pageContent = page.getDocumentElement();
575 HtmlElement div = pageContent.getElementById("sik");
576 List<HtmlElement> htmlElementsByTagName = div
577 .getHtmlElementsByTagName("a");
578
579 Region region = portalService.findRegionByName(anchor.asText());
580
581 for (HtmlElement element : htmlElementsByTagName) {
582 ImportMunicipal((HtmlAnchor) element, government, country,
583 region);
584 }
585 }
586 }
587
588 /***
589 * Import municipal.
590 *
591 * @param anchor the anchor
592 * @param government the government
593 * @param country the country
594 * @param region the region
595 * @throws Exception the exception
596 */
597 private void ImportMunicipal(HtmlAnchor anchor, Government government,
598 Country country, Region region) throws Exception {
599 HtmlPage page = (HtmlPage) anchor.openLinkInNewWindow();
600 HtmlElement pageContent = page.getDocumentElement();
601
602 Region local = new Region();
603 local.setParent(region);
604 region.getParts().add(local);
605 local.setRegionType(RegionType.Local);
606 local.setName(anchor.asText());
607
608
609 Government localGovernment = portalService.createRegionalGovernment(
610 anchor.asText().trim() + " Kommun", country, local,
611 OrganisationType.Local, government);
612
613 Election election = new Election();
614 election.setName(localGovernment.getName() + " val");
615 election.setType(ElectionType.Local);
616 election.setGovernment(localGovernment);
617 localGovernment.getElections().add(election);
618
619
620 portalService.createOrganisation("Kommunstyrelse " + local.getName(),
621 null, country, local, OrganisationType.Local, localGovernment);
622
623 Organisation localParliament = portalService.createOrganisation(
624 "Kommunfullmäktige " + local.getName(), null, country,
625 local, OrganisationType.Local, localGovernment);
626 Role fullmäktigeLedamot = portalService.createRoleInOrg(
627 "Fullmäktigeledamot", localParliament);
628
629 log.info("Importing members in " + localParliament.getName());
630
631 List<Politician> members = getAllMunicipialsMembers(pageContent);
632 for (Politician member : members) {
633 PoliticalParty politicalParty = portalService.findPartyByShortCode(member.getParty().toLowerCase().trim());
634
635 Person person = new Person();
636 person.setName(member.getName());
637
638
639 portalService.addMember(politicalParty, person);
640 portalService.addMember(localParliament, person);
641 portalService.addRolePlayed(fullmäktigeLedamot, person);
642 }
643 }
644
645 /***
646 * Gets the all municipials members.
647 *
648 * @param pageContent the page content
649 * @return the all municipials members
650 */
651 @SuppressWarnings("unchecked")
652 private List<Politician> getAllMunicipialsMembers(HtmlElement pageContent) {
653
654 HtmlTableBody tableBody = (HtmlTableBody) pageContent
655 .getElementById("tableBody");
656 List<HtmlTableRow> rows = tableBody.getRows();
657
658 List<Politician> members = new ArrayList<Politician>();
659 for (HtmlTableRow row : rows) {
660 List<HtmlTableCell> cells = row.getCells();
661
662 if (cells.size() == 3) {
663 members.add(new Politician(cells.get(0).asText().trim(), "",
664 cells.get(1).asText().trim(), cells.get(2).asText()
665 .trim(), "Fullmäktigeledamot"));
666 } else if (cells.size() == 2) {
667 members
668 .add(new Politician(cells.get(0).asText().trim(), "",
669 cells.get(1).asText().trim(), "",
670 "Fullmäktigeledamot"));
671 }
672 }
673 return members;
674 }
675
676 /***
677 * Gets the all regional government members.
678 *
679 * @param pageContent the page content
680 * @return the all regional government members
681 */
682 @SuppressWarnings( { "unchecked", "unchecked" })
683 private List<Politician> getAllRegionalGovernmentMembers(
684 HtmlElement pageContent) {
685
686 HtmlTableBody tableBody = (HtmlTableBody) pageContent
687 .getElementById("tableBody");
688 List<HtmlTableRow> rows = tableBody.getRows();
689
690 List<Politician> members = new ArrayList<Politician>();
691 for (HtmlTableRow row : rows) {
692 List<HtmlTableCell> cells = row.getCells();
693
694 if (cells.size() == 3) {
695 members.add(new Politician(cells.get(0).asText().trim(), "",
696 cells.get(1).asText().trim(), cells.get(2).asText()
697 .trim(), "Landstingsfullmäktige"));
698 } else if (cells.size() == 2) {
699 members.add(new Politician(cells.get(0).asText().trim(), "",
700 cells.get(1).asText().trim(), "",
701 "Landstingsfullmäktige"));
702 }
703 }
704 return members;
705 }
706
707 /***
708 * Import international governments.
709 *
710 * @param government the government
711 * @param country the country
712 * @throws Exception the exception
713 */
714 public void importInternationalGovernments(Government government,
715 Country country) throws Exception {
716 HtmlPage page = (HtmlPage) webClient
717 .getPage("http://www.val.se/val/ep2004/valdaled/index.html");
718 HtmlElement pageContent = page.getDocumentElement();
719
720 Region eu = new Region();
721 eu.setRegionType(RegionType.International);
722 eu.setName("Europeiska unionen");
723 country.setParent(eu);
724 eu.getParts().add(country);
725
726
727
728 Organisation internationalGov = portalService.createOrganisation(
729 "Europaparlamentet", null, country,
730 eu, OrganisationType.International, government);
731 Role parlamentsLedamot = portalService.createRoleInOrg(
732 "Parlamentsledamot", internationalGov);
733
734 List<Politician> members = getAllInternationalGovernmentMembers(pageContent);
735 for (Politician member : members) {
736 PoliticalParty politicalParty = portalService.findPartyByShortCode(member.getParty().toLowerCase().trim());
737
738 Person person = portalService.createPerson(member.getName());
739
740 portalService.addMemberById(politicalParty.getId(), person.getId());
741 portalService.addMemberById(internationalGov.getId(), person.getId());
742 portalService.addRolePlayedById(parlamentsLedamot.getId(), person.getId());
743 }
744 }
745
746 /***
747 * Gets the all international government members.
748 *
749 * @param pageContent the page content
750 * @return the all international government members
751 */
752 @SuppressWarnings("unchecked")
753 private List<Politician> getAllInternationalGovernmentMembers(
754 HtmlElement pageContent) {
755
756 HtmlTableBody tableBody = (HtmlTableBody) pageContent
757 .getElementById("tableBody");
758 List<HtmlTableRow> rows = tableBody.getRows();
759
760 List<Politician> members = new ArrayList<Politician>();
761 for (HtmlTableRow row : rows) {
762 List<HtmlTableCell> cells = row.getCells();
763
764 if (cells.size() == 3) {
765 members.add(new Politician(cells.get(0).asText().trim(), "",
766 cells.get(1).asText().trim(), cells.get(2).asText()
767 .trim(), "Parlamentsledamot"));
768 } else if (cells.size() == 2) {
769 members.add(new Politician(cells.get(0).asText().trim(), "",
770 cells.get(1).asText().trim(), "", "Parlamentsledamot"));
771 }
772 }
773 return members;
774 }
775 }